public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var shardKeyArrived = evnt.GetBody <ShardRangeArrived>();
TheTrace.TraceInformation("Got {0}->{1} from {2}", shardKeyArrived.InclusiveStartKey,
shardKeyArrived.InclusiveEndKey, shardKeyArrived.Source.TypeName);
var account = CloudStorageAccount.Parse(shardKeyArrived.Source.ConnectionString);
var client = account.CreateCloudTableClient();
var table = client.GetTableReference(shardKeyArrived.Source.DynamicProperties["TableName"].ToString());
var entities = table.ExecuteQuery(new TableQuery().Where(
TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", "ge", shardKeyArrived.InclusiveStartKey),
TableOperators.And,
TableQuery.GenerateFilterCondition("PartitionKey", "le", shardKeyArrived.InclusiveEndKey))));
bool hasAnything = false;
foreach (var entity in entities)
{
await _pusher.PushAsync(entity, shardKeyArrived.Source);
hasAnything = true;
}
if (hasAnything)
{
await _pusher.FlushAsync();
}
return(Enumerable.Empty <Event>());
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var shardKeyArrived = evnt.GetBody <ShardKeyArrived>();
TheTrace.TraceInformation("Got {0} from {1}", shardKeyArrived.ShardKey,
shardKeyArrived.Source.TypeName);
var shardKeyQuerier = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardKeyQuery);
var query = FactoryHelper.Create <IShardKeyQuery>(shardKeyQuerier, typeof(TableStorageShardKeyQuery));
var entities = await query.QueryAsync(shardKeyArrived);
bool hasAnything = false;
foreach (var entity in entities)
{
await _pusher.PushAsync(entity, shardKeyArrived.Source);
hasAnything = true;
}
if (hasAnything)
{
await _pusher.FlushAsync();
}
return(Enumerable.Empty <Event>());
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var shardKeyArrived = evnt.GetBody <ShardKeyArrived>();
_telemetryProvider.WriteTelemetry(
"ShardKey receive message delay duration",
(long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
shardKeyArrived.Source.TypeName);
await _durationInstrumentor.InstrumentAsync(async() =>
{
TheTrace.TraceInformation("Got {0} from {1}", shardKeyArrived.ShardKey,
shardKeyArrived.Source.TypeName);
var shardKeyQuerier = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardKeyQuery);
var query = FactoryHelper.Create <IShardKeyQuery>(shardKeyQuerier, typeof(TableStorageShardKeyQuery));
var entities = await query.QueryAsync(shardKeyArrived);
var minDateTime = DateTimeOffset.MaxValue;
var hasAnything = false;
int n = 0;
var shardKeyTime = shardKeyArrived.GetDateTimeOffset().ToString("yyyyMMddHHmm");
foreach (var entity in entities)
{
var eventDateTimeOffset = entity.GetEventDateTimeOffset();
var delayInSeconds = entity.Timestamp.Subtract(eventDateTimeOffset).TotalSeconds;
if (delayInSeconds >= _shardKeyDelayWarning)
{
TheTrace.TraceWarning("SHARD_KEY_ACTOR_DELAY_DETECTED => Delay of {0} seconds for {1} in shardKey {2} and time {3}",
delayInSeconds, shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
}
entity.Timestamp = eventDateTimeOffset;
await _pusher.PushAsync(entity, shardKeyArrived.Source);
hasAnything = true;
minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime;
n++;
}
TheTrace.TraceInformation("Gathered {0} records for {1} and ShardKey {2} => {1}_{2} {1}_{3}", n,
shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
if (hasAnything)
{
await _pusher.FlushAsync();
_telemetryProvider.WriteTelemetry(
"ShardKeyArrivedActor log delay duration",
(long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
shardKeyArrived.Source.TypeName);
}
});
return(Enumerable.Empty <Event>());
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var blobFileArrived = evnt.GetBody <BlobFileArrived>();
TheTrace.TraceInformation("Got {0} from {1}", blobFileArrived.BlobId,
blobFileArrived.Source.TypeName);
var account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference(blobFileArrived.Source.DynamicProperties["ContainerName"].ToString());
var uri = new Uri(blobFileArrived.BlobId);
var id = string.Join("", uri.Segments.Skip(2));
var blob = container.GetBlockBlobReference(id);
if (!blob.Exists())
{
throw new InvalidOperationException("Blob does not exist: " + id);
}
var stream = new MemoryStream();
await blob.DownloadToStreamAsync(stream);
var parserTypeName = blobFileArrived.Source.DynamicProperties["Parser"].ToString();
if (parserTypeName == null)
{
throw new NullReferenceException("parserTypeName");
}
var parserType = Assembly.GetExecutingAssembly().GetType(parserTypeName) ?? Type.GetType(parserTypeName);
if (parserType == null)
{
throw new InvalidOperationException("Parser type was null: " + parserTypeName);
}
var parser = (IParser)Activator.CreateInstance(parserType);
bool hasAnything = false;
foreach (var entity in parser.Parse(stream, blob.Uri))
{
await _pusher.PushAsync(entity, blobFileArrived.Source);
hasAnything = true;
}
if (hasAnything)
{
await _pusher.FlushAsync();
}
return(Enumerable.Empty <Event>());
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var blobFileArrived = evnt.GetBody <BlobFileArrived>();
_telemetryProvider.WriteTelemetry(
"BlobFileActor receive message delay duration",
(long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
blobFileArrived.Source.TypeName);
await _durationInstrumentor.InstrumentAsync(async() =>
{
TheTrace.TraceInformation("Got {0} from {1}", blobFileArrived.BlobId,
blobFileArrived.Source.TypeName);
var account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference(blobFileArrived.Source.DynamicProperties["ContainerName"].ToString());
var uri = new Uri(blobFileArrived.BlobId);
var id = string.Join("", uri.Segments.Skip(2));
var blob = container.GetBlockBlobReference(id);
if (!blob.Exists())
{
throw new InvalidOperationException("Blob does not exist: " + id);
}
var stream = new MemoryStream();
await blob.DownloadToStreamAsync(stream);
var parser = FactoryHelper.Create <IParser>(blobFileArrived.Source.DynamicProperties["Parser"].ToString(), typeof(IisLogParser));
var hasAnything = false;
var minDateTime = DateTimeOffset.MaxValue;
foreach (var entity in parser.Parse(stream, blob.Uri, blobFileArrived.Position ?? 0, blobFileArrived.EndPosition ?? 0))
{
await _pusher.PushAsync(entity, blobFileArrived.Source);
hasAnything = true;
minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime;
}
if (hasAnything)
{
await _pusher.FlushAsync();
_telemetryProvider.WriteTelemetry(
"BlobFileActor log delay duration",
(long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
blobFileArrived.Source.TypeName);
}
}, blobFileArrived.Source.TypeName);
return(Enumerable.Empty <Event>());
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var blobFileArrived = evnt.GetBody <BlobFileArrived>();
TheTrace.TraceInformation("Got {0} from {1}", blobFileArrived.BlobId,
blobFileArrived.Source.TypeName);
var account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference(blobFileArrived.Source.DynamicProperties["ContainerName"].ToString());
var uri = new Uri(blobFileArrived.BlobId);
var id = string.Join("", uri.Segments.Skip(2));
var blob = container.GetBlockBlobReference(id);
if (!blob.Exists())
{
throw new InvalidOperationException("Blob does not exist: " + id);
}
var stream = new MemoryStream();
await blob.DownloadToStreamAsync(stream);
var parser = FactoryHelper.Create <IParser>(blobFileArrived.Source.DynamicProperties["Parser"].ToString(), typeof(IisLogParser));
bool hasAnything = false;
foreach (var entity in parser.Parse(stream, blob.Uri, blobFileArrived.Position ?? 0, blobFileArrived.EndPosition ?? 0))
{
await _pusher.PushAsync(entity, blobFileArrived.Source);
hasAnything = true;
}
if (hasAnything)
{
await _pusher.FlushAsync();
}
return(Enumerable.Empty <Event>());
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var events = new List <Event>();
var blobFileScheduled = evnt.GetBody <BlobFileScheduled>();
var account = CloudStorageAccount.Parse(blobFileScheduled.Source.ConnectionString);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference("wad-iis-logfiles");
bool nextBlobExists;
bool previousBlobExists;
bool mainBlobExists;
Func <string, string> alternater = s => s.Replace(".log", "_x.log");
var nextBlob = container.GetBlobReferenceWithAlternateName(blobFileScheduled.NextFile, alternater, out nextBlobExists);
var previousBlob = container.GetBlobReferenceWithAlternateName(blobFileScheduled.PreviousFile, alternater, out previousBlobExists);
var mainBlob = container.GetBlobReferenceWithAlternateName(blobFileScheduled.FileToConsume, alternater, out mainBlobExists);
if (!previousBlobExists && !mainBlobExists)
{
TheTrace.TraceInformation("BlobFileConventionActor - previous blob does not exist. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
return(events); // will never be here. Stop chasing it.
}
long currentLength = 0;
if (mainBlobExists)
{
currentLength = mainBlob.Properties.Length;
if (currentLength == blobFileScheduled.LastPosition)
{
if (nextBlobExists)
{
TheTrace.TraceInformation("BlobFileConventionActor - Next blob exists. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
return(events); // done and dusted. Stop chasing it.
}
if (blobFileScheduled.StopChasingAfter < DateTimeOffset.Now)
{
TheTrace.TraceInformation("BlobFileConventionActor - Chase time past. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
return(events); // Stop chasing it.
}
}
else
{
var stream = await DownloadToFileAsync(mainBlob);
try
{
currentLength = stream.Length;
var parser = FactoryHelper.Create <IParser>(blobFileScheduled.Source.DynamicProperties["Parser"].ToString(), typeof(IisLogParser));
bool hasAnything = false;
foreach (var entity in parser.Parse(stream, mainBlob.Uri, blobFileScheduled.LastPosition))
{
await _pusher.PushAsync(entity, blobFileScheduled.Source);
hasAnything = true;
}
if (hasAnything)
{
await _pusher.FlushAsync();
TheTrace.TraceInformation("BlobFileConventionActor - pushed records for {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
}
}
finally
{
stream.Close();
File.Delete(stream.Name);
}
}
}
else
{
if (blobFileScheduled.StopChasingAfter < DateTimeOffset.Now)
{
TheTrace.TraceInformation("BlobFileConventionActor - Chase time past. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
return(events); // Stop chasing it.
}
}
blobFileScheduled.LastPosition = currentLength;
// let's defer
events.Add(new Event(blobFileScheduled)
{
EnqueueTime = DateTimeOffset.Now.Add(TimeSpan.FromSeconds(30))
});
TheTrace.TraceInformation("BlobFileConventionActor - deferred processing {0}. Length => {1} at {2}", blobFileScheduled.FileToConsume, currentLength, DateTimeOffset.Now);
return(events);
}
