public static bool TryCreate(string cfg, string shorthand, Dictionary <string, string> parameters, out Process process) { var builder = new ContainerBuilder(); builder.RegisterModule(new RootModule(shorthand)); builder.Register <IPipelineLogger>(c => new NLogPipelineLogger(SlugifyTransform.Slugify(cfg))).As <IPipelineLogger>().SingleInstance(); builder.Register <IContext>(c => new PipelineContext(c.Resolve <IPipelineLogger>())).As <IContext>(); using (var scope = builder.Build().BeginLifetimeScope()) { process = scope.Resolve <Process>(new NamedParameter("cfg", cfg), new NamedParameter("parameters", parameters)); var context = scope.Resolve <IContext>(); foreach (var warning in process.Warnings()) { context.Warn(warning); } if (process.Errors().Any()) { foreach (var error in process.Errors()) { context.Error(error); } context.Error("The configuration errors must be fixed before this job will run."); } else { process.Preserve = true; } } return(process.Errors().Length == 0); }
public void Execute(Process process) { var logger = _logger ?? new NLogPipelineLogger(SlugifyTransform.Slugify(Cfg)); if (process.OutputIsConsole()) { logger.SuppressConsole(); } if (_checkMemory) { CheckMemory(process, logger); } using (var scope = DefaultContainer.Create(process, logger, Options.PlaceHolderStyle)) { try { scope.Resolve <IProcessController>().Execute(); } catch (Exception ex) { var context = scope.Resolve <IContext>(); context.Error(ex.Message); context.Logger.Clear(); new LibaryVersionChecker(context).Check(); } } }
public static bool TryCreate(Options options, Dictionary <string, string> parameters, out Process process) { var logger = new NLogPipelineLogger(SlugifyTransform.Slugify(options.Arrangement)); using (var scope = ConfigurationContainer.Create(options.Arrangement, logger, parameters, options.PlaceHolderStyle)) { process = scope.Resolve <Process>(); var context = new PipelineContext(logger, process); foreach (var warning in process.Warnings()) { context.Warn(warning); } if (process.Errors().Any()) { foreach (var error in process.Errors()) { context.Error(error); } context.Error("The configuration errors must be fixed before this job will run."); } else { process.Preserve = true; } } return(process != null && process.Errors().Length == 0); }
public void Execute(string cfg, string shorthand, Dictionary <string, string> parameters) { var builder = new ContainerBuilder(); builder.RegisterModule(new RootModule(shorthand)); builder.Register <IPipelineLogger>(c => new NLogPipelineLogger(SlugifyTransform.Slugify(cfg))).As <IPipelineLogger>().SingleInstance(); builder.Register <IContext>(c => new PipelineContext(c.Resolve <IPipelineLogger>())).As <IContext>(); using (var scope = builder.Build().BeginLifetimeScope()) { var context = scope.Resolve <IContext>(); var process = scope.Resolve <Process>(new NamedParameter("cfg", cfg)); foreach (var warning in process.Warnings()) { context.Warn(warning); } if (process.Errors().Any()) { foreach (var error in process.Errors()) { context.Error(error); } context.Error("The configuration errors must be fixed before this job will run."); return; } _cfg = cfg; Execute(process); } }
public void Execute(Process process) { var logger = _logger ?? new NLogPipelineLogger(SlugifyTransform.Slugify(Cfg)); if (process.OutputIsConsole()) { logger.SuppressConsole(); } if (RunCount % 3 == 0) { if (_checkMemory) { CheckMemory(process, logger); } else { var context = new PipelineContext(logger, process); context.Info($"Process has run {RunCount} time{RunCount.Plural()}."); } } using (var scope = DefaultContainer.Create(process, logger, Options.PlaceHolderStyle)) { try { scope.Resolve <IProcessController>().Execute(); } catch (Exception ex) { var context = scope.Resolve <IContext>(); context.Error(ex.Message); context.Logger.Clear(); new LibaryVersionChecker(context).Check(); } } ++RunCount; }
protected override void Load(ContainerBuilder builder) { _options.Arrangement = _options.ArrangementWithMode(); builder.Register <IPipelineLogger>(c => new NLogPipelineLogger(SlugifyTransform.Slugify(_options.Arrangement))).As <IPipelineLogger>().SingleInstance(); builder.Register <IContext>(c => new PipelineContext(c.Resolve <IPipelineLogger>())).As <IContext>(); // for now scheduler builder.Register(c => new RunTimeSchemaReader(c.Resolve <IContext>())).As <IRunTimeSchemaReader>(); builder.Register <ISchemaHelper>(ctx => new SchemaHelper(ctx.Resolve <IContext>(), ctx.Resolve <IRunTimeSchemaReader>())).As <ISchemaHelper>(); // for quartz scheduler builder.Register <ILoggerFactoryAdapter>(ctx => new QuartzLogAdaptor(ctx.Resolve <IContext>(), Scheduler.Quartz.Utility.ConvertLevel(ctx.Resolve <IContext>().LogLevel), true, true, false, "o")).As <ILoggerFactoryAdapter>(); builder.Register(ctx => new QuartzJobFactory(_options, ctx.Resolve <IPipelineLogger>())).As <IJobFactory>().SingleInstance(); builder.Register <IScheduler>((ctx, p) => { if (string.IsNullOrEmpty(_options.Schedule) || _options.Mode != null && _options.Mode.In("init", "check")) { return(new NowScheduler(_options, ctx.Resolve <ISchemaHelper>(), ctx.Resolve <IPipelineLogger>())); } if (_options.Schedule == "internal") { var process = ProcessFactory.Create(_options.Arrangement, new Dictionary <string, string>()); if (process.Errors().Any()) { Console.Error.WriteLine("In order for an internal schedule to work, the arrangement passed in must be valid!"); foreach (var error in process.Errors()) { Console.Error.WriteLine(error); } Environment.Exit(1); } return(new QuartzCronSchedulerViaInternalSchedule(_options, process.Schedule, ctx.Resolve <IJobFactory>(), ctx.Resolve <ILoggerFactoryAdapter>())); } return(new QuartzCronSchedulerViaCommandLine(_options, ctx.Resolve <IJobFactory>(), ctx.Resolve <ILoggerFactoryAdapter>())); }).As <IScheduler>(); }
protected override void Load(ContainerBuilder builder) { _options.Arrangement = _options.ArrangementWithMode(); builder.Register <IPipelineLogger>(c => new NLogPipelineLogger(SlugifyTransform.Slugify(_options.Arrangement))).As <IPipelineLogger>().SingleInstance(); builder.Register <IContext>(c => new PipelineContext(c.Resolve <IPipelineLogger>())).As <IContext>(); // for now scheduler builder.Register(c => new RunTimeSchemaReader(c.Resolve <IContext>())).As <IRunTimeSchemaReader>(); builder.Register <ISchemaHelper>(ctx => new SchemaHelper(ctx.Resolve <IContext>(), ctx.Resolve <IRunTimeSchemaReader>())).As <ISchemaHelper>(); // for quartz scheduler builder.RegisterType <QuartzJobFactory>().As <IJobFactory>().SingleInstance(); builder.Register <ILoggerFactoryAdapter>((ctx => new QuartzLogAdaptor(ctx.Resolve <IContext>(), Scheduler.Quartz.Utility.ConvertLevel(ctx.Resolve <IContext>().LogLevel), true, true, false, "o"))).As <ILoggerFactoryAdapter>(); builder.Register <IScheduler>((ctx, p) => { if (string.IsNullOrEmpty(_options.Schedule) || _options.Mode != null && _options.Mode.In("init", "check")) { return(new NowScheduler(_options, ctx.Resolve <ISchemaHelper>())); } return(new QuartzCronScheduler(_options, ctx.Resolve <IJobFactory>(), ctx.Resolve <ILoggerFactoryAdapter>())); }).As <IScheduler>(); }
protected override void Load(ContainerBuilder builder) { // This reader is used to load the initial configuration and nested resources for tfl actions, etc. builder.RegisterType <FileReader>().Named <IReader>("file"); builder.RegisterType <WebReader>().Named <IReader>("web"); builder.Register <IReader>(ctx => new DefaultReader( ctx.ResolveNamed <IReader>("file"), new ReTryingReader(ctx.ResolveNamed <IReader>("web"), attempts: 3)) ); builder.Register((ctx, p) => { var placeHolderStyle = "@()"; if (ctx.IsRegisteredWithName <string>("placeHolderStyle")) { placeHolderStyle = ctx.ResolveNamed <string>("placeHolderStyle"); } var dependencies = new List <IDependency> { ctx.Resolve <IReader>(), new FormParameterModifier(new DateMathModifier()), new EnvironmentModifier(new PlaceHolderReplacer(placeHolderStyle[0], placeHolderStyle[1], placeHolderStyle[2])) }; if (ctx.IsRegisteredWithName <IDependency>("shorthand-t")) { dependencies.Add(ctx.ResolveNamed <IDependency>("shorthand-t")); } if (ctx.IsRegisteredWithName <IDependency>("shorthand-v")) { dependencies.Add(ctx.ResolveNamed <IDependency>("shorthand-v")); } var process = new Process(dependencies.ToArray()); switch (p.Count()) { case 2: process.Load( p.Named <string>("cfg"), p.Named <Dictionary <string, string> >("parameters") ); break; default: process.Load(p.Named <string>("cfg")); break; } if (process.Errors().Any()) { return(process); } // this might be put into it's own type and injected (or not) if (process.Entities.Count == 1) { var entity = process.Entities.First(); if (!entity.HasInput() && ctx.IsRegistered <ISchemaReader>()) { var schemaReader = ctx.Resolve <ISchemaReader>(new TypedParameter(typeof(Process), process)); var schema = schemaReader.Read(entity); var newEntity = schema.Entities.First(); foreach (var sf in newEntity.Fields.Where(f => f.Name == Constants.TflKey || f.Name == Constants.TflDeleted || f.Name == Constants.TflBatchId || f.Name == Constants.TflHashCode)) { sf.Alias = newEntity.Name + sf.Name; } process.Entities.Clear(); process.Entities.Add(newEntity); process.Connections.First(c => c.Name == newEntity.Connection).Delimiter = schema.Connection.Delimiter; } } if (process.Output().Provider == Constants.DefaultSetting || process.Output().Provider == "internal") { try { Console.WindowHeight = Console.WindowHeight + 1 - 1; Console.Title = process.Name; if (!System.Environment.CommandLine.Contains("TESTWINDOW") && !System.Environment.CommandLine.Contains("TESTPLATFORM")) { process.Output().Provider = "console"; } } catch (Exception) { // just a hack to determine if in console } } // handling multiple entities with non-relational output var originalOutput = process.Output().Clone(); originalOutput.Name = Constants.OriginalOutput; originalOutput.Key = process.Name + originalOutput.Name; if (!process.OutputIsRelational() && (process.Entities.Count > 1 || process.Buffer)) { process.Output().Provider = process.InternalProvider; var folder = Path.Combine(System.Environment.GetFolderPath(System.Environment.SpecialFolder.LocalApplicationData), Constants.ApplicationFolder); var file = new FileInfo(Path.Combine(folder, SlugifyTransform.Slugify(process.Name) + "." + (process.InternalProvider == "sqlce" ? "sdf" : "sqlite3"))); var exists = file.Exists; process.Output().File = file.FullName; process.Output().RequestTimeout = process.InternalProvider == "sqlce" ? 0 : process.Output().RequestTimeout; process.Flatten = process.InternalProvider == "sqlce"; process.Mode = exists ? process.Mode : "init"; process.Connections.Add(originalOutput); if (!exists) { if (!Directory.Exists(file.DirectoryName)) { Directory.CreateDirectory(folder); } } } return(process); }).As <Process>().InstancePerDependency(); // because it has state, if you run it again, it's not so good }
public void Start() { var builder = new ContainerBuilder(); builder.Register <IPipelineLogger>(c => new NLogPipelineLogger(SlugifyTransform.Slugify(_options.Arrangement))).As <IPipelineLogger>().SingleInstance(); builder.RegisterModule(new RootModule(_options.Shorthand)); builder.Register <IContext>(c => new PipelineContext(c.Resolve <IPipelineLogger>())).As <IContext>(); builder.Register(c => new NowExecutor(c.Resolve <IPipelineLogger>(), _options.Arrangement, _options.Shorthand, _options.Mode)).As <IRunTimeExecute>(); using (var scope = builder.Build().BeginLifetimeScope()) { var context = scope.Resolve <IContext>(); var process = scope.Resolve <Configuration.Process>( new NamedParameter("cfg", _options.Arrangement) ); foreach (var warning in process.Warnings()) { context.Debug(() => warning); } if (process.Errors().Any()) { foreach (var error in process.Errors()) { context.Error(error); } context.Error("The configuration errors must be fixed before this job will run."); context.Logger.Clear(); return; } if (process.Entities.Any(e => process.Connections.First(c => c.Name == e.Connection).Provider != "internal" && !e.Fields.Any(f => f.Input))) { context.Debug(() => "Detecting schema..."); if (_schemaHelper.Help(process)) { if (process.Errors().Any()) { foreach (var error in process.Errors()) { context.Error(error); } context.Error("The configuration errors must be fixed before this job will run."); context.Logger.Clear(); return; } } } if (_options.Mode != null && _options.Mode.ToLower() == "check") { SimplifyForOutput(process); Console.WriteLine(process.Serialize()); return; } if (_options.Mode != "default") { process.Mode = _options.Mode; } scope.Resolve <IRunTimeExecute>().Execute(process); } }
public void Start() { var builder = new ContainerBuilder(); builder.Register <IPipelineLogger>(c => new NLogPipelineLogger(SlugifyTransform.Slugify(_options.Arrangement))).As <IPipelineLogger>().SingleInstance(); builder.RegisterModule(new RootModule(_options.Shorthand)); builder.Register <IContext>(c => new PipelineContext(c.Resolve <IPipelineLogger>())).As <IContext>(); builder.Register(c => new RunTimeExecutor(_options.Arrangement, _options.Mode, _options.Format)).As <IRunTimeExecute>(); using (var scope = builder.Build().BeginLifetimeScope()) { var context = scope.Resolve <IContext>(); var process = scope.Resolve <Configuration.Process>( new NamedParameter("cfg", _options.Arrangement) ); foreach (var warning in process.Warnings()) { context.Warn(warning); } if (process.Errors().Any()) { foreach (var error in process.Errors()) { context.Error(error); } context.Error("The configuration errors must be fixed before this job will run."); return; } if (process.Entities.Any(e => !e.Fields.Any(f => f.Input))) { context.Debug(() => "Detecting schema..."); if (_schemaHelper.Help(process)) { if (process.Errors().Any()) { foreach (var error in process.Errors()) { context.Error(error); } context.Error("The configuration errors must be fixed before this job will run."); return; } } } if (_options.Mode != null && _options.Mode.ToLower() == "check") { process.Star = string.Empty; foreach (var connection in process.Connections) { connection.Delimiters.Clear(); } foreach (var entity in process.Entities) { entity.CalculateHashCode = true; if (entity.Name == entity.Alias) { entity.Alias = null; } entity.Fields.RemoveAll(f => f.System); } foreach (var field in process.GetAllFields().Where(f => !string.IsNullOrEmpty(f.T))) { field.T = string.Empty; } foreach (var field in process.GetAllFields()) { if (field.Name == field.Alias) { field.Alias = null; } if (field.Name == field.Label) { field.Label = string.Empty; } field.SortField = string.Empty; field.Sortable = Constants.DefaultSetting; } Console.WriteLine(process.Serialize()); return; } scope.Resolve <IRunTimeExecute>().Execute(process); } }
public void Execute(Process process) { process.Mode = _mode; // Since we're in a Console app, and honor output format if (process.Output().Provider.In("internal", "console")) { process.Output().Provider = "console"; process.Output().Format = _format; } var logger = new NLogPipelineLogger(SlugifyTransform.Slugify(_cfg)); if (!string.IsNullOrEmpty(process.MaxMemory)) { var context = new PipelineContext(logger, process); var timer = new Stopwatch(); timer.Start(); GC.Collect(); GC.WaitForPendingFinalizers(); timer.Stop(); context.Info($"Collected free memory. Time taken: {timer.Elapsed}."); var currentBytes = System.Diagnostics.Process.GetCurrentProcess().WorkingSet64.Bytes(); var maxMemory = ByteSize.Parse(process.MaxMemory); if (maxMemory.CompareTo(currentBytes) < 0) { context.Error($"Process exceeded {maxMemory.Megabytes.ToString("#.0")} Mb. Current memory is {currentBytes.Megabytes.ToString("#.0")} Mb!"); Environment.Exit(1); } else { context.Info($"The process is using {currentBytes.Megabytes.ToString("#.0")} Mb of it's max {maxMemory.Megabytes.ToString("#.0")} Mb allowed."); } } var builder = new ContainerBuilder(); builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance(); builder.RegisterCallback(new RootModule(process.Shorthand).Configure); builder.RegisterCallback(new ContextModule(process).Configure); // providers builder.RegisterCallback(new AdoModule(process).Configure); builder.RegisterCallback(new LuceneModule(process).Configure); builder.RegisterCallback(new SolrModule(process).Configure); builder.RegisterCallback(new ElasticModule(process).Configure); builder.RegisterCallback(new InternalModule(process).Configure); builder.RegisterCallback(new FileModule(process).Configure); builder.RegisterCallback(new FolderModule(process).Configure); builder.RegisterCallback(new DirectoryModule(process).Configure); builder.RegisterCallback(new ExcelModule(process).Configure); builder.RegisterCallback(new WebModule(process).Configure); builder.RegisterCallback(new MapModule(process).Configure); builder.RegisterCallback(new TemplateModule(process).Configure); builder.RegisterCallback(new ActionModule(process).Configure); builder.RegisterCallback(new EntityPipelineModule(process).Configure); builder.RegisterCallback(new ProcessPipelineModule(process).Configure); builder.RegisterCallback(new ProcessControlModule(process).Configure); using (var scope = builder.Build().BeginLifetimeScope()) { try { scope.Resolve <IProcessController>().Execute(); } catch (Exception ex) { scope.Resolve <IContext>().Error(ex.Message); } } }