public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var shardKeyArrived = evnt.GetBody <ShardKeyArrived>();
_telemetryProvider.WriteTelemetry(
"ShardKey receive message delay duration",
(long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
shardKeyArrived.Source.TypeName);
await _durationInstrumentor.InstrumentAsync(async() =>
{
TheTrace.TraceInformation("Got {0} from {1}", shardKeyArrived.ShardKey,
shardKeyArrived.Source.TypeName);
var shardKeyQuerier = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardKeyQuery);
var query = FactoryHelper.Create <IShardKeyQuery>(shardKeyQuerier, typeof(TableStorageShardKeyQuery));
var entities = await query.QueryAsync(shardKeyArrived);
var shardKeyTime = shardKeyArrived.GetDateTimeOffset().ToString("yyyyMMddHHmm");
await _pusher.PushAll(PreprocessEntities(entities, shardKeyArrived, shardKeyTime), shardKeyArrived.Source).ConfigureAwait(false);
}).ConfigureAwait(false);
return(Enumerable.Empty <Event>());
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var blobFileArrived = evnt.GetBody <BlobFileArrived>();
_telemetryProvider.WriteTelemetry(
"BlobFileActor receive message delay duration",
(long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
blobFileArrived.Source.TypeName);
await _durationInstrumentor.InstrumentAsync(async() =>
{
TheTrace.TraceInformation("Got {0} from {1}", blobFileArrived.BlobId,
blobFileArrived.Source.TypeName);
//var account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString);
CloudStorageAccount account;
if (!String.IsNullOrWhiteSpace(blobFileArrived.Source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(blobFileArrived.Source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
account = new CloudStorageAccount(accountSas, blobFileArrived.Source.AccountName, "", useHttps: true);
}
else
{
account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString);
}
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference(blobFileArrived.Source.DynamicProperties["ContainerName"].ToString());
var uri = new Uri(blobFileArrived.BlobId);
var id = string.Join("", uri.Segments.Skip(2));
var blob = container.GetBlockBlobReference(id);
if (!blob.Exists())
{
throw new InvalidOperationException("Blob does not exist: " + id);
}
var stream = new MemoryStream();
await blob.DownloadToStreamAsync(stream);
var parser = FactoryHelper.Create <IParser>(blobFileArrived.Source.DynamicProperties["Parser"].ToString(), typeof(IisLogParser));
var hasAnything = false;
var minDateTime = DateTimeOffset.UtcNow;
var records = parser.Parse(() => stream, blob.Uri, blobFileArrived.Source, new ParseCursor(blobFileArrived.Position ?? 0)
{
EndPosition = blobFileArrived.EndPosition ?? 0
});
var seenPages = await _pusher.PushAll(records, blobFileArrived.Source).ConfigureAwait(false);
hasAnything = seenPages > 0;
if (hasAnything)
{
_telemetryProvider.WriteTelemetry(
"BlobFileActor message processing duration",
(long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
blobFileArrived.Source.TypeName);
}
}, blobFileArrived.Source.TypeName);
return(Enumerable.Empty <Event>());
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var shardKeyArrived = evnt.GetBody <ShardKeyArrived>();
_telemetryProvider.WriteTelemetry(
"ShardKey receive message delay duration",
(long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
shardKeyArrived.Source.TypeName);
await _durationInstrumentor.InstrumentAsync(async() =>
{
TheTrace.TraceInformation("Got {0} from {1}", shardKeyArrived.ShardKey,
shardKeyArrived.Source.TypeName);
var shardKeyQuerier = (string)shardKeyArrived.Source.GetDynamicProperty(ConveyorBeltConstants.ShardKeyQuery);
var query = FactoryHelper.Create <IShardKeyQuery>(shardKeyQuerier, typeof(TableStorageShardKeyQuery));
var entities = await query.QueryAsync(shardKeyArrived);
var minDateTime = DateTimeOffset.MaxValue;
var hasAnything = false;
int n = 0;
var shardKeyTime = shardKeyArrived.GetDateTimeOffset().ToString("yyyyMMddHHmm");
foreach (var entity in entities)
{
var eventDateTimeOffset = entity.GetEventDateTimeOffset();
var delayInSeconds = entity.Timestamp.Subtract(eventDateTimeOffset).TotalSeconds;
if (delayInSeconds >= _shardKeyDelayWarning)
{
TheTrace.TraceWarning("SHARD_KEY_ACTOR_DELAY_DETECTED => Delay of {0} seconds for {1} in shardKey {2} and time {3}",
delayInSeconds, shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
}
entity.Timestamp = eventDateTimeOffset;
await _pusher.PushAsync(entity, shardKeyArrived.Source);
hasAnything = true;
minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime;
n++;
}
TheTrace.TraceInformation("Gathered {0} records for {1} and ShardKey {2} => {1}_{2} {1}_{3}", n,
shardKeyArrived.Source.TypeName, shardKeyArrived.ShardKey, shardKeyTime);
if (hasAnything)
{
await _pusher.FlushAsync();
_telemetryProvider.WriteTelemetry(
"ShardKeyArrivedActor log delay duration",
(long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
shardKeyArrived.Source.TypeName);
}
});
return(Enumerable.Empty <Event>());
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var blobFileArrived = evnt.GetBody <BlobFileArrived>();
_telemetryProvider.WriteTelemetry(
"BlobFileActor receive message delay duration",
(long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
blobFileArrived.Source.TypeName);
await _durationInstrumentor.InstrumentAsync(async() =>
{
TheTrace.TraceInformation("Got {0} from {1}", blobFileArrived.BlobId,
blobFileArrived.Source.TypeName);
var account = CloudStorageAccount.Parse(blobFileArrived.Source.ConnectionString);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference(blobFileArrived.Source.DynamicProperties["ContainerName"].ToString());
var uri = new Uri(blobFileArrived.BlobId);
var id = string.Join("", uri.Segments.Skip(2));
var blob = container.GetBlockBlobReference(id);
if (!blob.Exists())
{
throw new InvalidOperationException("Blob does not exist: " + id);
}
var stream = new MemoryStream();
await blob.DownloadToStreamAsync(stream);
var parser = FactoryHelper.Create <IParser>(blobFileArrived.Source.DynamicProperties["Parser"].ToString(), typeof(IisLogParser));
var hasAnything = false;
var minDateTime = DateTimeOffset.MaxValue;
foreach (var entity in parser.Parse(stream, blob.Uri, blobFileArrived.Position ?? 0, blobFileArrived.EndPosition ?? 0))
{
await _pusher.PushAsync(entity, blobFileArrived.Source);
hasAnything = true;
minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime;
}
if (hasAnything)
{
await _pusher.FlushAsync();
_telemetryProvider.WriteTelemetry(
"BlobFileActor log delay duration",
(long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
blobFileArrived.Source.TypeName);
}
}, blobFileArrived.Source.TypeName);
return(Enumerable.Empty <Event>());
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var events = new List <Event>();
var blobFileScheduled = evnt.GetBody <BlobFileScheduled>();
_telemetryProvider.WriteTelemetry(
"BlobFileScheduled receive message delay duration",
(long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
blobFileScheduled.Source.TypeName);
await _durationInstrumentor.InstrumentAsync(async() =>
{
var account = CloudStorageAccount.Parse(blobFileScheduled.Source.ConnectionString);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference("wad-iis-logfiles");
bool nextBlobExists;
bool previousBlobExists;
bool mainBlobExists;
Func <string, string> alternater = s => s.Replace(".log", "_x.log");
container.GetBlobReferenceWithAlternateName(blobFileScheduled.NextFile, alternater, out nextBlobExists);
container.GetBlobReferenceWithAlternateName(blobFileScheduled.PreviousFile, alternater, out previousBlobExists);
var mainBlob = container.GetBlobReferenceWithAlternateName(blobFileScheduled.FileToConsume, alternater, out mainBlobExists);
if (!previousBlobExists && !mainBlobExists)
{
TheTrace.TraceInformation("BlobFileConventionActor - previous blob does not exist. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
return; // will never be here. Stop chasing it.
}
long currentLength = 0;
if (mainBlobExists)
{
currentLength = mainBlob.Properties.Length;
if (currentLength == blobFileScheduled.LastPosition)
{
if (nextBlobExists)
{
TheTrace.TraceInformation("BlobFileConventionActor - Next blob exists. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
return; // done and dusted. Stop chasing it.
}
if (blobFileScheduled.StopChasingAfter < DateTimeOffset.Now)
{
TheTrace.TraceInformation("BlobFileConventionActor - Chase time past. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
return; // Stop chasing it.
}
}
else
{
var stream = await DownloadToFileAsync(mainBlob);
try
{
currentLength = stream.Length;
var parser = FactoryHelper.Create <IParser>(blobFileScheduled.Source.DynamicProperties["Parser"].ToString(), typeof(IisBlobConventionScheduler));
var hasAnything = false;
var minDateTime = DateTimeOffset.MaxValue;
foreach (var entity in parser.Parse(stream, mainBlob.Uri, blobFileScheduled.LastPosition))
{
await _pusher.PushAsync(entity, blobFileScheduled.Source);
hasAnything = true;
minDateTime = minDateTime > entity.Timestamp ? entity.Timestamp : minDateTime;
}
if (hasAnything)
{
await _pusher.FlushAsync();
TheTrace.TraceInformation("BlobFileConventionActor - pushed records for {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
_telemetryProvider.WriteTelemetry(
"BlobFileConventionActor log delay duration",
(long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds,
blobFileScheduled.Source.TypeName);
}
}
finally
{
stream.Close();
File.Delete(stream.Name);
}
}
}
else
{
if (blobFileScheduled.StopChasingAfter < DateTimeOffset.Now)
{
TheTrace.TraceInformation("BlobFileConventionActor - Chase time past. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
return; // Stop chasing it.
}
}
blobFileScheduled.LastPosition = currentLength;
// let's defer
events.Add(new Event(blobFileScheduled)
{
EnqueueTime = DateTimeOffset.Now.Add(TimeSpan.FromSeconds(30))
});
TheTrace.TraceInformation("BlobFileConventionActor - deferred processing {0}. Length => {1} at {2}", blobFileScheduled.FileToConsume, currentLength, DateTimeOffset.Now);
});
return(events);
}
private async Task <DiagnosticsSource> TryScheduleSourceAsync(DiagnosticsSource source)
{
try
{
source = _sourceConfiguration.RefreshSource(source);
TheTrace.TraceInformation("MasterScheduler - Scheduling {0}", source.ToTypeKey());
if (!source.IsActive.HasValue || !source.IsActive.Value)
{
TheTrace.TraceInformation("MasterScheduler - NOT active: {0}", source.ToTypeKey());
return(null);
}
var createMappings = _configurationValueProvider.GetValue(ConfigurationKeys.EsCreateMappings);
if (Convert.ToBoolean(createMappings))
{
await SetupMappingsAsync(source);
}
if (!source.LastScheduled.HasValue)
{
source.LastScheduled = DateTimeOffset.UtcNow.AddDays(-1);
}
// if has been recently scheduled
if (source.LastScheduled.Value.AddMinutes(source.SchedulingFrequencyMinutes.Value) >
DateTimeOffset.UtcNow)
{
TheTrace.TraceInformation("MasterScheduler - Nothing to do with {0}. LastScheduled in Future {1}",
source.ToTypeKey(), source.LastScheduled.Value);
return(null);
}
_telemetryProvider.WriteTelemetry(
"MasterScheduler duration since last scheduled",
(long)(DateTime.UtcNow - source.LastScheduled).Value.TotalMilliseconds,
source.ToTypeKey());
var schedulerType = Assembly.GetExecutingAssembly().GetType(source.SchedulerType) ??
Type.GetType(source.SchedulerType);
if (schedulerType == null)
{
source.ErrorMessage = "Could not find SchedulerType: " + source.SchedulerType;
}
else
{
await _scheduleDurationInstrumentor.InstrumentAsync(async() =>
{
var scheduler = (ISourceScheduler)_locator.GetService(schedulerType);
var result = await scheduler.TryScheduleAsync(source);
source.LastScheduled = DateTimeOffset.UtcNow;
TheTrace.TraceInformation(
"MasterScheduler - Got result for TryScheduleAsync in {0}. Success => {1}",
source.ToTypeKey(), result.Item1);
if (result.Item2)
{
await _eventQueueOperator.PushBatchAsync(result.Item1);
}
source.ErrorMessage = string.Empty;
TheTrace.TraceInformation("MasterScheduler - Finished Scheduling {0}", source.ToTypeKey());
}, source.ToTypeKey());
}
return(source);
}
catch (Exception e)
{
TheTrace.TraceError(e.ToString());
source.ErrorMessage = e.ToString();
return(source);
}
}
public async Task <IEnumerable <Event> > ProcessAsync(Event evnt)
{
var events = new List <Event>();
var blobFileScheduled = evnt.GetBody <BlobFileScheduled>();
_telemetryProvider.WriteTelemetry(
"BlobFileScheduled receive message delay duration",
(long)(DateTime.UtcNow - evnt.Timestamp).TotalMilliseconds,
blobFileScheduled.Source.TypeName);
await _durationInstrumentor.InstrumentAsync(async() =>
{
//var account = CloudStorageAccount.Parse(blobFileScheduled.Source.ConnectionString);
CloudStorageAccount account;
if (!String.IsNullOrWhiteSpace(blobFileScheduled.Source.AccountSasKey))
{
// Create new storage credentials using the SAS token.
var accountSas = new StorageCredentials(blobFileScheduled.Source.AccountSasKey);
// Use these credentials and the account name to create a Blob service client.
account = new CloudStorageAccount(accountSas, blobFileScheduled.Source.AccountName, "", useHttps: true);
}
else
{
account = CloudStorageAccount.Parse(blobFileScheduled.Source.ConnectionString);
}
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference("wad-iis-logfiles");
bool nextBlobExists;
bool previousBlobExists;
bool mainBlobExists;
Func <string, string> alternater = s => s.Replace(".log", "_x.log");
container.GetBlobReferenceWithAlternateName(blobFileScheduled.NextFile, alternater, out nextBlobExists);
container.GetBlobReferenceWithAlternateName(blobFileScheduled.PreviousFile, alternater, out previousBlobExists);
var mainBlob = container.GetBlobReferenceWithAlternateName(blobFileScheduled.FileToConsume, alternater, out mainBlobExists);
if (!previousBlobExists && !mainBlobExists)
{
TheTrace.TraceInformation("BlobFileConventionActor - previous blob does not exist. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
return; // will never be here. Stop chasing it.
}
long currentLength = 0;
long currentOffset = 0;
if (mainBlobExists)
{
currentLength = mainBlob.Properties.Length;
currentOffset = blobFileScheduled.LastPosition;
if (currentLength > blobFileScheduled.LastPosition)
{
var parser = FactoryHelper.Create <IParser>(blobFileScheduled.Source.DynamicProperties["Parser"].ToString(), typeof(IisLogParser));
var minDateTime = DateTimeOffset.UtcNow;
var cursor = new ParseCursor(blobFileScheduled.LastPosition);
var parsedRecords = parser.Parse(() => StreamFactory(mainBlob), mainBlob.Uri, blobFileScheduled.Source, cursor);
var pages = await _pusher.PushAll(parsedRecords, blobFileScheduled.Source).ConfigureAwait(false);
currentOffset = cursor.EndPosition;
if (pages > 0)
{
TheTrace.TraceInformation("BlobFileConventionActor - pushed records for {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
_telemetryProvider.WriteTelemetry("BlobFileConventionActor log delay duration", (long)(DateTimeOffset.UtcNow - minDateTime).TotalMilliseconds, blobFileScheduled.Source.TypeName);
}
}
currentLength = mainBlob.Properties.Length;
if (nextBlobExists)
{
TheTrace.TraceInformation("BlobFileConventionActor - Next blob exists. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
return; // done and dusted. Stop chasing it.
}
}
if (blobFileScheduled.StopChasingAfter < DateTimeOffset.Now)
{
TheTrace.TraceInformation("BlobFileConventionActor - Chase time past. Stopped chasing {0} at {1}", blobFileScheduled.FileToConsume, DateTimeOffset.Now);
return; // Stop chasing it.
}
if (currentOffset == blobFileScheduled.LastPosition && currentOffset < blobFileScheduled.LastPosition)
{
throw new Exception("Did nothing but did not reach EOF");
}
blobFileScheduled.LastPosition = currentOffset;
// let's defer
events.Add(new Event(blobFileScheduled)
{
EnqueueTime = DateTimeOffset.Now.Add(TimeSpan.FromSeconds(30))
});
TheTrace.TraceInformation("BlobFileConventionActor - deferred processing {0}. Length => {1} at {2}", blobFileScheduled.FileToConsume, currentLength, DateTimeOffset.Now);
});
return(events);
}