internal static AdlArchive FromConfig(ExecutionContext ctx, IConfigurationRoot config)
{
var clientId = config["adlClientId"];
var clientSecret = config["adlClientSecret"];
var tenantId = config["adlTenantId"];
var path = config["adlPath"];
var creds = new ClientCredential(clientId, clientSecret);
var token = ApplicationTokenProvider.LoginSilentAsync(tenantId, creds).Result;
var archive = new AdlArchive();
archive.Client = AdlsClient.CreateClient(path, token);
archive.Ctx = ctx;
archive.Root = config["adlRoot"];
archive.AdlList = config["adlList"];
return(archive);
}
private static void Main(string[] args)
{
const string configFile = "appsettings.json";
var config = new ConfigurationBuilder()
.AddJsonFile(configFile, true, true)
.Build();
var ctx = ExecutionContext.FromConfig(config);
var archive = AdlArchive.FromConfig(ctx, config);
var producer = new ArchiveEventProducer(ctx, archive);
var consumer = ArchiveEventConsumer.FromConfig(ctx, config);
var watermarkStorage = new FileWatermarkStorage();
var watermarkManagers = new Dictionary <int, PartitionWatermarkManager>();
foreach (var item in producer.ProduceItems())
{
if (ctx.Noisy)
{
Console.WriteLine($"processing {item}...");
}
PartitionWatermarkManager manager = null;
if (!watermarkManagers.ContainsKey(item.Partition))
{
manager = new PartitionWatermarkManager(watermarkStorage, item.Partition);
watermarkManagers[item.Partition] = manager;
}
manager = watermarkManagers[item.Partition];
if (!manager.ShouldProceedWith(item, null))
{
if (ctx.Noisy)
{
Console.WriteLine("Skipping as below the watermark");
}
}
else
{
var eventsPerItem = 0;
long sizePerItem = 0;
var itemStart = DateTime.UtcNow;
var batch = new List <ArchiveEvent>();
foreach (var ev in producer.ProduceEvents(item))
{
var skipEvent = ctx.Boundaries != null &&
item.Partition <ctx.Boundaries.Count &&
ev.SequenceNumber> ctx.Boundaries[item.Partition];
if (!skipEvent)
{
eventsPerItem++;
sizePerItem += ev.Body.Length;
batch.Add(ev);
}
else
{
if (ctx.Noisy)
{
Console.WriteLine(
$"Skipping sequence number {ev.SequenceNumber} for partiton {item.Partition} as higher than the boundary");
}
}
}
if (!ctx.DryRun)
{
Func <ArchiveItem, List <ArchiveEvent>, PartitionWatermarkManager, bool> tryConsumeFunc =
consumer.TryConsumeBatch;
if (ctx.SendIndividually)
{
tryConsumeFunc = consumer.TryConsumeBatchIndividually;
}
if (!tryConsumeFunc(item, batch, manager))
{
if (ctx.Noisy)
{
Console.WriteLine("Failed to send events to the Event Hub");
}
}
}
if (ctx.Noisy)
{
if (eventsPerItem == 0)
{
Console.WriteLine("Item size too small");
}
else
{
Console.WriteLine($"events: {eventsPerItem}, size: {sizePerItem}");
var secs = (DateTime.UtcNow - itemStart).TotalSeconds;
Console.WriteLine($"time per item: {secs} sec");
Console.WriteLine($"Throughput: {eventsPerItem / secs} events/sec");
Console.WriteLine($"Throughput: {sizePerItem / secs} bytes/sec");
}
}
}
}
}