public bool TryConsumeBatchIndividually(ArchiveItem item, List <ArchiveEvent> events,
PartitionWatermarkManager watermarkManager)
{
foreach (var ev in events)
{
if (!watermarkManager.ShouldProceedWith(item, ev))
{
if (Ctx.Noisy)
{
Console.WriteLine($"skipping sequenceNumber {ev.SequenceNumber} as below the watermark");
}
continue;
}
var eventData = new EventData(ev.Body);
foreach (var entry in ev.Properties)
{
eventData.Properties[entry.Key] = entry.Value;
}
try
{
EventHubClient.SendAsync(eventData, item.Partition.ToString()).Wait();
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
return(false);
}
watermarkManager.UpdateWatermark(item, ev);
}
return(true);
}
private static void Main(string[] args)
{
const string configFile = "appsettings.json";
var config = new ConfigurationBuilder()
.AddJsonFile(configFile, true, true)
.Build();
var ctx = ExecutionContext.FromConfig(config);
var archive = AdlArchive.FromConfig(ctx, config);
var producer = new ArchiveEventProducer(ctx, archive);
var consumer = ArchiveEventConsumer.FromConfig(ctx, config);
var watermarkStorage = new FileWatermarkStorage();
var watermarkManagers = new Dictionary <int, PartitionWatermarkManager>();
foreach (var item in producer.ProduceItems())
{
if (ctx.Noisy)
{
Console.WriteLine($"processing {item}...");
}
PartitionWatermarkManager manager = null;
if (!watermarkManagers.ContainsKey(item.Partition))
{
manager = new PartitionWatermarkManager(watermarkStorage, item.Partition);
watermarkManagers[item.Partition] = manager;
}
manager = watermarkManagers[item.Partition];
if (!manager.ShouldProceedWith(item, null))
{
if (ctx.Noisy)
{
Console.WriteLine("Skipping as below the watermark");
}
}
else
{
var eventsPerItem = 0;
long sizePerItem = 0;
var itemStart = DateTime.UtcNow;
var batch = new List <ArchiveEvent>();
foreach (var ev in producer.ProduceEvents(item))
{
var skipEvent = ctx.Boundaries != null &&
item.Partition <ctx.Boundaries.Count &&
ev.SequenceNumber> ctx.Boundaries[item.Partition];
if (!skipEvent)
{
eventsPerItem++;
sizePerItem += ev.Body.Length;
batch.Add(ev);
}
else
{
if (ctx.Noisy)
{
Console.WriteLine(
$"Skipping sequence number {ev.SequenceNumber} for partiton {item.Partition} as higher than the boundary");
}
}
}
if (!ctx.DryRun)
{
Func <ArchiveItem, List <ArchiveEvent>, PartitionWatermarkManager, bool> tryConsumeFunc =
consumer.TryConsumeBatch;
if (ctx.SendIndividually)
{
tryConsumeFunc = consumer.TryConsumeBatchIndividually;
}
if (!tryConsumeFunc(item, batch, manager))
{
if (ctx.Noisy)
{
Console.WriteLine("Failed to send events to the Event Hub");
}
}
}
if (ctx.Noisy)
{
if (eventsPerItem == 0)
{
Console.WriteLine("Item size too small");
}
else
{
Console.WriteLine($"events: {eventsPerItem}, size: {sizePerItem}");
var secs = (DateTime.UtcNow - itemStart).TotalSeconds;
Console.WriteLine($"time per item: {secs} sec");
Console.WriteLine($"Throughput: {eventsPerItem / secs} events/sec");
Console.WriteLine($"Throughput: {sizePerItem / secs} bytes/sec");
}
}
}
}
}
