protected override async Task RunAsync(CancellationToken cancellationToken)
{
int batchSize = 5;
long delayMs = 100;
DockerPipelineRunnerOptions options = new DockerPipelineRunnerOptions();
var block = new ActionBlock <DocumentRequest>(async(@event) =>
{
var services = new ServiceCollection();
// using (var fa =)
{
services.AddLogging(builder =>
{
builder.AddSerilog(new LoggerConfiguration()
.Enrich.WithProperty("ApiVersion", "1.2.5000")
.WriteTo.File($"c:/logs/{@event.id}/default.log", buffered: true)
.CreateLogger());
//builder.AddFile(o =>
//{
// o.FileAppender = new PhysicalFileAppender(new PhysicalFileProvider("c:/logs"));
// o.BasePath = @event?.id;
// o.EnsureBasePath = true;
// // o.FallbackFileName = $"{@event?.id}.log";
//});
});
// create logger factory
using (var sp = services.BuildServiceProvider())
{
var loggerFactory = sp.GetService <ILoggerFactory>();
var logger = loggerFactory.CreateLogger <PipelineExecutorService>();
try
{
logger.LogInformation("Received<{eventId}>: {@event}", @event.id, @event.data);
var args = @event.data.arguments;
var ci = new DockerClientExecutor(
new DockerClientConfiguration(new Uri("npipe://./pipe/docker_engine"))
.CreateClient(), loggerFactory.CreateLogger <DockerClientExecutor>());
var runner = new DockerPipelineRunner(
loggerFactory.CreateLogger <DockerPipelineRunner>(),
new ExpressionParser(@event.data.pipeline
.UpdateParametersFromConsoleArguments(args), loggerFactory.CreateLogger <ExpressionParser>())
.AddRegex()
.AddSplit()
.AddConcat()
.AddAll()
, ci, options
);
await runner.RunPipelineAsync(CancellationToken.None);
}
catch (Exception ex)
{
Console.WriteLine(ex);
logger.LogError(ex, "Failed to run pipeline");
}
}
}
// await connection.InvokeAsync("pipeline_recieved", arg1: @event.id);
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = batchSize,
BoundedCapacity = batchSize
});
while (!cancellationToken.IsCancellationRequested)
{
var md5 = MD5.Create();
var value = md5.ComputeHash(Encoding.ASCII.GetBytes(Guid.NewGuid().ToString()));
var key = BitConverter.ToInt64(value, 0);
var pipelineRunnerService = ServiceProxy.Create <IPipelineRunnerService>(
new Uri($"fabric:/EarthML.PipelinesApplication/PipelineRunnerService"),
partitionKey: new ServicePartitionKey(key), listenerName: "V2_1Listener");
var items = await pipelineRunnerService.GetBatchAsync(batchSize, cancellationToken);
foreach (var item in items)
{
await block.SendAsync(item);
}
int delayFactor = batchSize - items.Length;
await Task.Delay(TimeSpan.FromMilliseconds(delayMs *delayFactor), cancellationToken);
}
block.Complete();
await block.Completion;
}
private async Task <int> OnExecuteAsync(CommandLineApplication app)
{
Thread.CurrentThread.CurrentCulture = CultureInfo.GetCultureInfo("en-US");
if (!string.IsNullOrEmpty(logpath))
{
Directory.CreateDirectory(logpath);
}
//var cc = new DockerClientConfiguration(new Uri("npipe://./pipe/docker_engine"))
// .CreateClient();
//var list = await cc.Volumes.ListAsync();
//var t = await cc.Volumes.InspectAsync(list.Volumes.First().Name);
//return 0;
//Console.WriteLine(new Uri("unix://var/run/docker.sock"));
if (Daemon)
{
DockerPipelineRunnerOptions options = new DockerPipelineRunnerOptions();
var test = new DockerClientConfiguration(new Uri(dockerSock))
.CreateClient();
var images = await test.Containers.ListContainersAsync(new Docker.DotNet.Models.ContainersListParameters());
foreach (var image in images)
{
Console.WriteLine(image.Image);
}
var connected = true;
var connection = new HubConnectionBuilder()
.WithUrl("https://dockerpipeline.azurewebsites.net/pipelines")
.ConfigureLogging(c =>
{
})
// .WithConsoleLogger()
.Build();
var block = new ActionBlock <DocumentRequest>(async(@event) =>
{
try
{
Console.Write(JsonConvert.SerializeObject(@event));
Console.WriteLine($"Received<{@event?.id}>: \n{@event?.data?.pipeline.ToString(Newtonsoft.Json.Formatting.Indented)}");
var args = @event.data.arguments;
var loggerFactory = new LoggerFactory();
var logger = loggerFactory.CreateLogger(this.GetType());
var ci = new DockerClientExecutor(new DockerClientConfiguration(new Uri(dockerSock))
.CreateClient(), logger);
var runner = new DockerPipelineRunner(logger,
new ExpressionParser(@event.data.pipeline
.UpdateParametersFromConsoleArguments(args), logger)
.AddRegex()
.AddSplit()
.AddConcat()
.AddAll()
, ci, options
);
await runner.RunPipelineAsync(CancellationToken.None);
}catch (Exception ex)
{
Console.WriteLine(ex);
}
await connection.InvokeAsync("pipeline_recieved", arg1: @event.id);
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = parallel
});
connection.On <DocumentRequest>("run_pipeline", async @event =>
{
await block.SendAsync(@event);
});
connection.Closed += (ex) => { connected = false; block.Complete(); return(Task.CompletedTask); };
await connection.StartAsync();
await connection.InvokeAsync("Connected", arg1 : images.Select(c => c.ID).ToArray());
await block.Completion;
return(0);
}
//var args = new[] {
// "--pipeline", "examples/sen2cor.json",
// "--id", "S2B_MSIL1C_20180130T071129_N0206_R106_T39RXL_20180130T082434",
// "--storageAccountKey", "z6GbmMTOfzK3w4Z6Cwd5acts1n327qbjWSLAviOhWMzSChA+Q9R/l/Sf5FgjvxhMBnaW4BMTby5Vwk+slmElGQ==",
// "--storageAccountName", "eodata",
// "--shareName", "uzjcx01tsuwxq18ymde4mdezmfqwnzexmjlftjaymdzfujewnl9umzlswexf",
// "--shareName1", "test2",
// "--registryServer","eodata.azurecr.io",
// "--registryPassword", "NvmS6ajuc8Htw3Sg",
// "--registryUsername" , "30beed1d-f53a-4fb0-a626-730c035a56be"};
//var ci = new DockerContainerInstanceExecutor(new DockerContainerInstanceExecutorOptions
//{
// SubscriptionId= "4c6c1ea4-3605-4411-a44e-b65382627f5b",
// ClientCredentials = new ClientCredential("http://AzureContainerInstanceRunner", "NvmS6ajuc8Htw3Sg"),
// ResourceGroup= "sentinel-playground",
// ContainerGroupName = "sen2cor8gb-test7",
// Cpu=1,
// MemoryInGB=8,
// OsType= "Linux",
// Location= "WestEurope",TenantId= "0840c760-6f7b-4556-b337-8c090e2d458d"
//});
// Console.WriteLine(await ci.ExecuteStepAsync(null, null, null, null, default));
// return;
{
var args =
app.Arguments.Select(c => new[] { $"--{c.Name}", c.Value })
.Concat(app.Options.Where(o => o.HasValue()).Select(o => new[] { $"--{o.LongName}", o.Value() }))
.SelectMany(m => m).Concat(app.RemainingArguments).ToArray();
var loggerFactory = new LoggerFactory();
var logger = loggerFactory.CreateLogger(this.GetType());
var ci = new DockerClientExecutor(new DockerClientConfiguration(new Uri(dockerSock))
.CreateClient(), logger);
var runner = new DockerPipelineRunner(logger,
new ExpressionParser(args.ReadAsDocument()
.UpdateParametersFromConsoleArguments(args), logger)
.AddRegex()
.AddSplit()
.AddConcat()
.AddAll()
, ci
);
await runner.RunPipelineAsync(CancellationToken.None);
}
return(0);
}