protected override async Task RunAsync(CancellationToken cancellationToken) { int batchSize = 5; long delayMs = 100; DockerPipelineRunnerOptions options = new DockerPipelineRunnerOptions(); var block = new ActionBlock <DocumentRequest>(async(@event) => { var services = new ServiceCollection(); // using (var fa =) { services.AddLogging(builder => { builder.AddSerilog(new LoggerConfiguration() .Enrich.WithProperty("ApiVersion", "1.2.5000") .WriteTo.File($"c:/logs/{@event.id}/default.log", buffered: true) .CreateLogger()); //builder.AddFile(o => //{ // o.FileAppender = new PhysicalFileAppender(new PhysicalFileProvider("c:/logs")); // o.BasePath = @event?.id; // o.EnsureBasePath = true; // // o.FallbackFileName = $"{@event?.id}.log"; //}); }); // create logger factory using (var sp = services.BuildServiceProvider()) { var loggerFactory = sp.GetService <ILoggerFactory>(); var logger = loggerFactory.CreateLogger <PipelineExecutorService>(); try { logger.LogInformation("Received<{eventId}>: {@event}", @event.id, @event.data); var args = @event.data.arguments; var ci = new DockerClientExecutor( new DockerClientConfiguration(new Uri("npipe://./pipe/docker_engine")) .CreateClient(), loggerFactory.CreateLogger <DockerClientExecutor>()); var runner = new DockerPipelineRunner( loggerFactory.CreateLogger <DockerPipelineRunner>(), new ExpressionParser(@event.data.pipeline .UpdateParametersFromConsoleArguments(args), loggerFactory.CreateLogger <ExpressionParser>()) .AddRegex() .AddSplit() .AddConcat() .AddAll() , ci, options ); await runner.RunPipelineAsync(CancellationToken.None); } catch (Exception ex) { Console.WriteLine(ex); logger.LogError(ex, "Failed to run pipeline"); } } } // await connection.InvokeAsync("pipeline_recieved", arg1: @event.id); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = batchSize, BoundedCapacity = batchSize }); while (!cancellationToken.IsCancellationRequested) { var md5 = MD5.Create(); var value = md5.ComputeHash(Encoding.ASCII.GetBytes(Guid.NewGuid().ToString())); var key = BitConverter.ToInt64(value, 0); var pipelineRunnerService = ServiceProxy.Create <IPipelineRunnerService>( new Uri($"fabric:/EarthML.PipelinesApplication/PipelineRunnerService"), partitionKey: new ServicePartitionKey(key), listenerName: "V2_1Listener"); var items = await pipelineRunnerService.GetBatchAsync(batchSize, cancellationToken); foreach (var item in items) { await block.SendAsync(item); } int delayFactor = batchSize - items.Length; await Task.Delay(TimeSpan.FromMilliseconds(delayMs *delayFactor), cancellationToken); } block.Complete(); await block.Completion; }
private async Task <int> OnExecuteAsync(CommandLineApplication app) { Thread.CurrentThread.CurrentCulture = CultureInfo.GetCultureInfo("en-US"); if (!string.IsNullOrEmpty(logpath)) { Directory.CreateDirectory(logpath); } //var cc = new DockerClientConfiguration(new Uri("npipe://./pipe/docker_engine")) // .CreateClient(); //var list = await cc.Volumes.ListAsync(); //var t = await cc.Volumes.InspectAsync(list.Volumes.First().Name); //return 0; //Console.WriteLine(new Uri("unix://var/run/docker.sock")); if (Daemon) { DockerPipelineRunnerOptions options = new DockerPipelineRunnerOptions(); var test = new DockerClientConfiguration(new Uri(dockerSock)) .CreateClient(); var images = await test.Containers.ListContainersAsync(new Docker.DotNet.Models.ContainersListParameters()); foreach (var image in images) { Console.WriteLine(image.Image); } var connected = true; var connection = new HubConnectionBuilder() .WithUrl("https://dockerpipeline.azurewebsites.net/pipelines") .ConfigureLogging(c => { }) // .WithConsoleLogger() .Build(); var block = new ActionBlock <DocumentRequest>(async(@event) => { try { Console.Write(JsonConvert.SerializeObject(@event)); Console.WriteLine($"Received<{@event?.id}>: \n{@event?.data?.pipeline.ToString(Newtonsoft.Json.Formatting.Indented)}"); var args = @event.data.arguments; var loggerFactory = new LoggerFactory(); var logger = loggerFactory.CreateLogger(this.GetType()); var ci = new DockerClientExecutor(new DockerClientConfiguration(new Uri(dockerSock)) .CreateClient(), logger); var runner = new DockerPipelineRunner(logger, new ExpressionParser(@event.data.pipeline .UpdateParametersFromConsoleArguments(args), logger) .AddRegex() .AddSplit() .AddConcat() .AddAll() , ci, options ); await runner.RunPipelineAsync(CancellationToken.None); }catch (Exception ex) { Console.WriteLine(ex); } await connection.InvokeAsync("pipeline_recieved", arg1: @event.id); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = parallel }); connection.On <DocumentRequest>("run_pipeline", async @event => { await block.SendAsync(@event); }); connection.Closed += (ex) => { connected = false; block.Complete(); return(Task.CompletedTask); }; await connection.StartAsync(); await connection.InvokeAsync("Connected", arg1 : images.Select(c => c.ID).ToArray()); await block.Completion; return(0); } //var args = new[] { // "--pipeline", "examples/sen2cor.json", // "--id", "S2B_MSIL1C_20180130T071129_N0206_R106_T39RXL_20180130T082434", // "--storageAccountKey", "z6GbmMTOfzK3w4Z6Cwd5acts1n327qbjWSLAviOhWMzSChA+Q9R/l/Sf5FgjvxhMBnaW4BMTby5Vwk+slmElGQ==", // "--storageAccountName", "eodata", // "--shareName", "uzjcx01tsuwxq18ymde4mdezmfqwnzexmjlftjaymdzfujewnl9umzlswexf", // "--shareName1", "test2", // "--registryServer","eodata.azurecr.io", // "--registryPassword", "NvmS6ajuc8Htw3Sg", // "--registryUsername" , "30beed1d-f53a-4fb0-a626-730c035a56be"}; //var ci = new DockerContainerInstanceExecutor(new DockerContainerInstanceExecutorOptions //{ // SubscriptionId= "4c6c1ea4-3605-4411-a44e-b65382627f5b", // ClientCredentials = new ClientCredential("http://AzureContainerInstanceRunner", "NvmS6ajuc8Htw3Sg"), // ResourceGroup= "sentinel-playground", // ContainerGroupName = "sen2cor8gb-test7", // Cpu=1, // MemoryInGB=8, // OsType= "Linux", // Location= "WestEurope",TenantId= "0840c760-6f7b-4556-b337-8c090e2d458d" //}); // Console.WriteLine(await ci.ExecuteStepAsync(null, null, null, null, default)); // return; { var args = app.Arguments.Select(c => new[] { $"--{c.Name}", c.Value }) .Concat(app.Options.Where(o => o.HasValue()).Select(o => new[] { $"--{o.LongName}", o.Value() })) .SelectMany(m => m).Concat(app.RemainingArguments).ToArray(); var loggerFactory = new LoggerFactory(); var logger = loggerFactory.CreateLogger(this.GetType()); var ci = new DockerClientExecutor(new DockerClientConfiguration(new Uri(dockerSock)) .CreateClient(), logger); var runner = new DockerPipelineRunner(logger, new ExpressionParser(args.ReadAsDocument() .UpdateParametersFromConsoleArguments(args), logger) .AddRegex() .AddSplit() .AddConcat() .AddAll() , ci ); await runner.RunPipelineAsync(CancellationToken.None); } return(0); }