public bool TryConsumeBatchIndividually(ArchiveItem item, List <ArchiveEvent> events,
                                                PartitionWatermarkManager watermarkManager)
        {
            foreach (var ev in events)
            {
                if (!watermarkManager.ShouldProceedWith(item, ev))
                {
                    if (Ctx.Noisy)
                    {
                        Console.WriteLine($"skipping sequenceNumber {ev.SequenceNumber} as below the watermark");
                    }
                    continue;
                }

                var eventData = new EventData(ev.Body);
                foreach (var entry in ev.Properties)
                {
                    eventData.Properties[entry.Key] = entry.Value;
                }
                try
                {
                    EventHubClient.SendAsync(eventData, item.Partition.ToString()).Wait();
                }
                catch (Exception e)
                {
                    Console.WriteLine(e.ToString());
                    return(false);
                }

                watermarkManager.UpdateWatermark(item, ev);
            }

            return(true);
        }
Exemple #2
0
        private static void Main(string[] args)
        {
            const string configFile = "appsettings.json";
            var          config     = new ConfigurationBuilder()
                                      .AddJsonFile(configFile, true, true)
                                      .Build();
            var ctx               = ExecutionContext.FromConfig(config);
            var archive           = AdlArchive.FromConfig(ctx, config);
            var producer          = new ArchiveEventProducer(ctx, archive);
            var consumer          = ArchiveEventConsumer.FromConfig(ctx, config);
            var watermarkStorage  = new FileWatermarkStorage();
            var watermarkManagers = new Dictionary <int, PartitionWatermarkManager>();


            foreach (var item in producer.ProduceItems())
            {
                if (ctx.Noisy)
                {
                    Console.WriteLine($"processing {item}...");
                }
                PartitionWatermarkManager manager = null;
                if (!watermarkManagers.ContainsKey(item.Partition))
                {
                    manager = new PartitionWatermarkManager(watermarkStorage, item.Partition);
                    watermarkManagers[item.Partition] = manager;
                }

                manager = watermarkManagers[item.Partition];
                if (!manager.ShouldProceedWith(item, null))
                {
                    if (ctx.Noisy)
                    {
                        Console.WriteLine("Skipping as below the watermark");
                    }
                }
                else

                {
                    var  eventsPerItem = 0;
                    long sizePerItem   = 0;
                    var  itemStart     = DateTime.UtcNow;
                    var  batch         = new List <ArchiveEvent>();
                    foreach (var ev in producer.ProduceEvents(item))

                    {
                        var skipEvent = ctx.Boundaries != null &&
                                        item.Partition <ctx.Boundaries.Count &&
                                                        ev.SequenceNumber> ctx.Boundaries[item.Partition];
                        if (!skipEvent)
                        {
                            eventsPerItem++;
                            sizePerItem += ev.Body.Length;
                            batch.Add(ev);
                        }
                        else
                        {
                            if (ctx.Noisy)
                            {
                                Console.WriteLine(
                                    $"Skipping sequence number {ev.SequenceNumber} for partiton {item.Partition} as higher than the boundary");
                            }
                        }
                    }


                    if (!ctx.DryRun)
                    {
                        Func <ArchiveItem, List <ArchiveEvent>, PartitionWatermarkManager, bool> tryConsumeFunc =
                            consumer.TryConsumeBatch;
                        if (ctx.SendIndividually)
                        {
                            tryConsumeFunc = consumer.TryConsumeBatchIndividually;
                        }


                        if (!tryConsumeFunc(item, batch, manager))
                        {
                            if (ctx.Noisy)
                            {
                                Console.WriteLine("Failed to send events to the Event Hub");
                            }
                        }
                    }

                    if (ctx.Noisy)
                    {
                        if (eventsPerItem == 0)
                        {
                            Console.WriteLine("Item size too small");
                        }
                        else
                        {
                            Console.WriteLine($"events: {eventsPerItem}, size: {sizePerItem}");
                            var secs = (DateTime.UtcNow - itemStart).TotalSeconds;
                            Console.WriteLine($"time per item: {secs} sec");
                            Console.WriteLine($"Throughput: {eventsPerItem / secs} events/sec");
                            Console.WriteLine($"Throughput: {sizePerItem / secs} bytes/sec");
                        }
                    }
                }
            }
        }