Esempio n. 1
0
        protected override async Task RunAsync(CancellationToken cancellationToken)
        {
            int  batchSize = 5;
            long delayMs   = 100;


            DockerPipelineRunnerOptions options = new DockerPipelineRunnerOptions();

            var block = new ActionBlock <DocumentRequest>(async(@event) =>
            {
                var services = new ServiceCollection();
                //  using (var fa =)
                {
                    services.AddLogging(builder =>
                    {
                        builder.AddSerilog(new LoggerConfiguration()
                                           .Enrich.WithProperty("ApiVersion", "1.2.5000")
                                           .WriteTo.File($"c:/logs/{@event.id}/default.log", buffered: true)
                                           .CreateLogger());

                        //builder.AddFile(o =>
                        //{
                        //    o.FileAppender = new PhysicalFileAppender(new PhysicalFileProvider("c:/logs"));
                        //    o.BasePath = @event?.id;
                        //    o.EnsureBasePath = true;
                        //   // o.FallbackFileName = $"{@event?.id}.log";
                        //});
                    });

                    // create logger factory
                    using (var sp = services.BuildServiceProvider())
                    {
                        var loggerFactory = sp.GetService <ILoggerFactory>();

                        var logger = loggerFactory.CreateLogger <PipelineExecutorService>();

                        try
                        {
                            logger.LogInformation("Received<{eventId}>: {@event}", @event.id, @event.data);



                            var args = @event.data.arguments;


                            var ci = new DockerClientExecutor(
                                new DockerClientConfiguration(new Uri("npipe://./pipe/docker_engine"))
                                .CreateClient(), loggerFactory.CreateLogger <DockerClientExecutor>());


                            var runner = new DockerPipelineRunner(
                                loggerFactory.CreateLogger <DockerPipelineRunner>(),
                                new ExpressionParser(@event.data.pipeline
                                                     .UpdateParametersFromConsoleArguments(args), loggerFactory.CreateLogger <ExpressionParser>())
                                .AddRegex()
                                .AddSplit()
                                .AddConcat()
                                .AddAll()
                                , ci, options
                                );



                            await runner.RunPipelineAsync(CancellationToken.None);
                        }
                        catch (Exception ex)
                        {
                            Console.WriteLine(ex);
                            logger.LogError(ex, "Failed to run pipeline");
                        }
                    }
                }

                //  await connection.InvokeAsync("pipeline_recieved", arg1: @event.id);
            }, new ExecutionDataflowBlockOptions
            {
                MaxDegreeOfParallelism = batchSize,
                BoundedCapacity        = batchSize
            });


            while (!cancellationToken.IsCancellationRequested)
            {
                var md5   = MD5.Create();
                var value = md5.ComputeHash(Encoding.ASCII.GetBytes(Guid.NewGuid().ToString()));
                var key   = BitConverter.ToInt64(value, 0);

                var pipelineRunnerService = ServiceProxy.Create <IPipelineRunnerService>(
                    new Uri($"fabric:/EarthML.PipelinesApplication/PipelineRunnerService"),
                    partitionKey: new ServicePartitionKey(key), listenerName: "V2_1Listener");

                var items = await pipelineRunnerService.GetBatchAsync(batchSize, cancellationToken);

                foreach (var item in items)
                {
                    await block.SendAsync(item);
                }

                int delayFactor = batchSize - items.Length;
                await Task.Delay(TimeSpan.FromMilliseconds(delayMs *delayFactor), cancellationToken);
            }

            block.Complete();
            await block.Completion;
        }
Esempio n. 2
0
        private async Task <int> OnExecuteAsync(CommandLineApplication app)
        {
            Thread.CurrentThread.CurrentCulture = CultureInfo.GetCultureInfo("en-US");

            if (!string.IsNullOrEmpty(logpath))
            {
                Directory.CreateDirectory(logpath);
            }
            //var cc = new DockerClientConfiguration(new Uri("npipe://./pipe/docker_engine"))
            //         .CreateClient();

            //var list = await cc.Volumes.ListAsync();
            //var t = await cc.Volumes.InspectAsync(list.Volumes.First().Name);

            //return 0;

            //Console.WriteLine(new Uri("unix://var/run/docker.sock"));


            if (Daemon)
            {
                DockerPipelineRunnerOptions options = new DockerPipelineRunnerOptions();

                var test = new DockerClientConfiguration(new Uri(dockerSock))
                           .CreateClient();


                var images = await test.Containers.ListContainersAsync(new Docker.DotNet.Models.ContainersListParameters());

                foreach (var image in images)
                {
                    Console.WriteLine(image.Image);
                }


                var connected  = true;
                var connection = new HubConnectionBuilder()
                                 .WithUrl("https://dockerpipeline.azurewebsites.net/pipelines")
                                 .ConfigureLogging(c =>
                {
                })
                                 //  .WithConsoleLogger()
                                 .Build();

                var block = new ActionBlock <DocumentRequest>(async(@event) =>
                {
                    try
                    {
                        Console.Write(JsonConvert.SerializeObject(@event));
                        Console.WriteLine($"Received<{@event?.id}>: \n{@event?.data?.pipeline.ToString(Newtonsoft.Json.Formatting.Indented)}");

                        var args = @event.data.arguments;

                        var loggerFactory = new LoggerFactory();
                        var logger        = loggerFactory.CreateLogger(this.GetType());

                        var ci = new DockerClientExecutor(new DockerClientConfiguration(new Uri(dockerSock))
                                                          .CreateClient(), logger);


                        var runner = new DockerPipelineRunner(logger,
                                                              new ExpressionParser(@event.data.pipeline
                                                                                   .UpdateParametersFromConsoleArguments(args), logger)
                                                              .AddRegex()
                                                              .AddSplit()
                                                              .AddConcat()
                                                              .AddAll()
                                                              , ci, options
                                                              );



                        await runner.RunPipelineAsync(CancellationToken.None);
                    }catch (Exception ex)
                    {
                        Console.WriteLine(ex);
                    }

                    await connection.InvokeAsync("pipeline_recieved", arg1: @event.id);
                }, new ExecutionDataflowBlockOptions
                {
                    MaxDegreeOfParallelism = parallel
                });


                connection.On <DocumentRequest>("run_pipeline", async @event =>
                {
                    await block.SendAsync(@event);
                });

                connection.Closed += (ex) => { connected = false; block.Complete(); return(Task.CompletedTask); };

                await connection.StartAsync();

                await connection.InvokeAsync("Connected", arg1 : images.Select(c => c.ID).ToArray());

                await block.Completion;

                return(0);
            }



            //var args = new[] {
            //    "--pipeline", "examples/sen2cor.json",
            //    "--id", "S2B_MSIL1C_20180130T071129_N0206_R106_T39RXL_20180130T082434",
            //    "--storageAccountKey", "z6GbmMTOfzK3w4Z6Cwd5acts1n327qbjWSLAviOhWMzSChA+Q9R/l/Sf5FgjvxhMBnaW4BMTby5Vwk+slmElGQ==",
            //    "--storageAccountName", "eodata",
            //    "--shareName", "uzjcx01tsuwxq18ymde4mdezmfqwnzexmjlftjaymdzfujewnl9umzlswexf",
            //    "--shareName1", "test2",
            //    "--registryServer","eodata.azurecr.io",
            //    "--registryPassword", "NvmS6ajuc8Htw3Sg",
            //    "--registryUsername" , "30beed1d-f53a-4fb0-a626-730c035a56be"};

            //var ci = new DockerContainerInstanceExecutor(new DockerContainerInstanceExecutorOptions
            //{
            //    SubscriptionId= "4c6c1ea4-3605-4411-a44e-b65382627f5b",
            //    ClientCredentials = new ClientCredential("http://AzureContainerInstanceRunner", "NvmS6ajuc8Htw3Sg"),
            //    ResourceGroup= "sentinel-playground",
            //    ContainerGroupName = "sen2cor8gb-test7",
            //    Cpu=1,
            //    MemoryInGB=8,
            //    OsType= "Linux",
            //    Location= "WestEurope",TenantId= "0840c760-6f7b-4556-b337-8c090e2d458d"
            //});
            //  Console.WriteLine(await ci.ExecuteStepAsync(null, null, null, null, default));
            // return;
            {
                var args =
                    app.Arguments.Select(c => new[] { $"--{c.Name}", c.Value })
                    .Concat(app.Options.Where(o => o.HasValue()).Select(o => new[] { $"--{o.LongName}", o.Value() }))
                    .SelectMany(m => m).Concat(app.RemainingArguments).ToArray();

                var loggerFactory = new LoggerFactory();
                var logger        = loggerFactory.CreateLogger(this.GetType());

                var ci = new DockerClientExecutor(new DockerClientConfiguration(new Uri(dockerSock))
                                                  .CreateClient(), logger);


                var runner = new DockerPipelineRunner(logger,
                                                      new ExpressionParser(args.ReadAsDocument()
                                                                           .UpdateParametersFromConsoleArguments(args), logger)
                                                      .AddRegex()
                                                      .AddSplit()
                                                      .AddConcat()
                                                      .AddAll()
                                                      , ci
                                                      );



                await runner.RunPipelineAsync(CancellationToken.None);
            }

            return(0);
        }