Exemple #1
0
        internal static AdlArchive FromConfig(ExecutionContext ctx, IConfigurationRoot config)
        {
            var clientId     = config["adlClientId"];
            var clientSecret = config["adlClientSecret"];
            var tenantId     = config["adlTenantId"];
            var path         = config["adlPath"];

            var creds   = new ClientCredential(clientId, clientSecret);
            var token   = ApplicationTokenProvider.LoginSilentAsync(tenantId, creds).Result;
            var archive = new AdlArchive();

            archive.Client  = AdlsClient.CreateClient(path, token);
            archive.Ctx     = ctx;
            archive.Root    = config["adlRoot"];
            archive.AdlList = config["adlList"];


            return(archive);
        }
Exemple #2
0
        private static void Main(string[] args)
        {
            const string configFile = "appsettings.json";
            var          config     = new ConfigurationBuilder()
                                      .AddJsonFile(configFile, true, true)
                                      .Build();
            var ctx               = ExecutionContext.FromConfig(config);
            var archive           = AdlArchive.FromConfig(ctx, config);
            var producer          = new ArchiveEventProducer(ctx, archive);
            var consumer          = ArchiveEventConsumer.FromConfig(ctx, config);
            var watermarkStorage  = new FileWatermarkStorage();
            var watermarkManagers = new Dictionary <int, PartitionWatermarkManager>();


            foreach (var item in producer.ProduceItems())
            {
                if (ctx.Noisy)
                {
                    Console.WriteLine($"processing {item}...");
                }
                PartitionWatermarkManager manager = null;
                if (!watermarkManagers.ContainsKey(item.Partition))
                {
                    manager = new PartitionWatermarkManager(watermarkStorage, item.Partition);
                    watermarkManagers[item.Partition] = manager;
                }

                manager = watermarkManagers[item.Partition];
                if (!manager.ShouldProceedWith(item, null))
                {
                    if (ctx.Noisy)
                    {
                        Console.WriteLine("Skipping as below the watermark");
                    }
                }
                else

                {
                    var  eventsPerItem = 0;
                    long sizePerItem   = 0;
                    var  itemStart     = DateTime.UtcNow;
                    var  batch         = new List <ArchiveEvent>();
                    foreach (var ev in producer.ProduceEvents(item))

                    {
                        var skipEvent = ctx.Boundaries != null &&
                                        item.Partition <ctx.Boundaries.Count &&
                                                        ev.SequenceNumber> ctx.Boundaries[item.Partition];
                        if (!skipEvent)
                        {
                            eventsPerItem++;
                            sizePerItem += ev.Body.Length;
                            batch.Add(ev);
                        }
                        else
                        {
                            if (ctx.Noisy)
                            {
                                Console.WriteLine(
                                    $"Skipping sequence number {ev.SequenceNumber} for partiton {item.Partition} as higher than the boundary");
                            }
                        }
                    }


                    if (!ctx.DryRun)
                    {
                        Func <ArchiveItem, List <ArchiveEvent>, PartitionWatermarkManager, bool> tryConsumeFunc =
                            consumer.TryConsumeBatch;
                        if (ctx.SendIndividually)
                        {
                            tryConsumeFunc = consumer.TryConsumeBatchIndividually;
                        }


                        if (!tryConsumeFunc(item, batch, manager))
                        {
                            if (ctx.Noisy)
                            {
                                Console.WriteLine("Failed to send events to the Event Hub");
                            }
                        }
                    }

                    if (ctx.Noisy)
                    {
                        if (eventsPerItem == 0)
                        {
                            Console.WriteLine("Item size too small");
                        }
                        else
                        {
                            Console.WriteLine($"events: {eventsPerItem}, size: {sizePerItem}");
                            var secs = (DateTime.UtcNow - itemStart).TotalSeconds;
                            Console.WriteLine($"time per item: {secs} sec");
                            Console.WriteLine($"Throughput: {eventsPerItem / secs} events/sec");
                            Console.WriteLine($"Throughput: {sizePerItem / secs} bytes/sec");
                        }
                    }
                }
            }
        }