public static bool TryCreate(string cfg, string shorthand, Dictionary <string, string> parameters, out Process process)
{
var builder = new ContainerBuilder();
builder.RegisterModule(new RootModule(shorthand));
builder.Register <IPipelineLogger>(c => new NLogPipelineLogger(SlugifyTransform.Slugify(cfg))).As <IPipelineLogger>().SingleInstance();
builder.Register <IContext>(c => new PipelineContext(c.Resolve <IPipelineLogger>())).As <IContext>();
using (var scope = builder.Build().BeginLifetimeScope()) {
process = scope.Resolve <Process>(new NamedParameter("cfg", cfg), new NamedParameter("parameters", parameters));
var context = scope.Resolve <IContext>();
foreach (var warning in process.Warnings())
{
context.Warn(warning);
}
if (process.Errors().Any())
{
foreach (var error in process.Errors())
{
context.Error(error);
}
context.Error("The configuration errors must be fixed before this job will run.");
}
else
{
process.Preserve = true;
}
}
return(process.Errors().Length == 0);
}
public void Execute(Process process)
{
var logger = _logger ?? new NLogPipelineLogger(SlugifyTransform.Slugify(Cfg));
if (process.OutputIsConsole())
{
logger.SuppressConsole();
}
if (_checkMemory)
{
CheckMemory(process, logger);
}
using (var scope = DefaultContainer.Create(process, logger, Options.PlaceHolderStyle)) {
try {
scope.Resolve <IProcessController>().Execute();
} catch (Exception ex) {
var context = scope.Resolve <IContext>();
context.Error(ex.Message);
context.Logger.Clear();
new LibaryVersionChecker(context).Check();
}
}
}
public static bool TryCreate(Options options, Dictionary <string, string> parameters, out Process process)
{
var logger = new NLogPipelineLogger(SlugifyTransform.Slugify(options.Arrangement));
using (var scope = ConfigurationContainer.Create(options.Arrangement, logger, parameters, options.PlaceHolderStyle)) {
process = scope.Resolve <Process>();
var context = new PipelineContext(logger, process);
foreach (var warning in process.Warnings())
{
context.Warn(warning);
}
if (process.Errors().Any())
{
foreach (var error in process.Errors())
{
context.Error(error);
}
context.Error("The configuration errors must be fixed before this job will run.");
}
else
{
process.Preserve = true;
}
}
return(process != null && process.Errors().Length == 0);
}
public void Execute(string cfg, string shorthand, Dictionary <string, string> parameters)
{
var builder = new ContainerBuilder();
builder.RegisterModule(new RootModule(shorthand));
builder.Register <IPipelineLogger>(c => new NLogPipelineLogger(SlugifyTransform.Slugify(cfg))).As <IPipelineLogger>().SingleInstance();
builder.Register <IContext>(c => new PipelineContext(c.Resolve <IPipelineLogger>())).As <IContext>();
using (var scope = builder.Build().BeginLifetimeScope()) {
var context = scope.Resolve <IContext>();
var process = scope.Resolve <Process>(new NamedParameter("cfg", cfg));
foreach (var warning in process.Warnings())
{
context.Warn(warning);
}
if (process.Errors().Any())
{
foreach (var error in process.Errors())
{
context.Error(error);
}
context.Error("The configuration errors must be fixed before this job will run.");
return;
}
_cfg = cfg;
Execute(process);
}
}
public void Execute(Process process)
{
var logger = _logger ?? new NLogPipelineLogger(SlugifyTransform.Slugify(Cfg));
if (process.OutputIsConsole())
{
logger.SuppressConsole();
}
if (RunCount % 3 == 0)
{
if (_checkMemory)
{
CheckMemory(process, logger);
}
else
{
var context = new PipelineContext(logger, process);
context.Info($"Process has run {RunCount} time{RunCount.Plural()}.");
}
}
using (var scope = DefaultContainer.Create(process, logger, Options.PlaceHolderStyle)) {
try {
scope.Resolve <IProcessController>().Execute();
} catch (Exception ex) {
var context = scope.Resolve <IContext>();
context.Error(ex.Message);
context.Logger.Clear();
new LibaryVersionChecker(context).Check();
}
}
++RunCount;
}
protected override void Load(ContainerBuilder builder)
{
_options.Arrangement = _options.ArrangementWithMode();
builder.Register <IPipelineLogger>(c => new NLogPipelineLogger(SlugifyTransform.Slugify(_options.Arrangement))).As <IPipelineLogger>().SingleInstance();
builder.Register <IContext>(c => new PipelineContext(c.Resolve <IPipelineLogger>())).As <IContext>();
// for now scheduler
builder.Register(c => new RunTimeSchemaReader(c.Resolve <IContext>())).As <IRunTimeSchemaReader>();
builder.Register <ISchemaHelper>(ctx => new SchemaHelper(ctx.Resolve <IContext>(), ctx.Resolve <IRunTimeSchemaReader>())).As <ISchemaHelper>();
// for quartz scheduler
builder.Register <ILoggerFactoryAdapter>(ctx => new QuartzLogAdaptor(ctx.Resolve <IContext>(), Scheduler.Quartz.Utility.ConvertLevel(ctx.Resolve <IContext>().LogLevel), true, true, false, "o")).As <ILoggerFactoryAdapter>();
builder.Register(ctx => new QuartzJobFactory(_options, ctx.Resolve <IPipelineLogger>())).As <IJobFactory>().SingleInstance();
builder.Register <IScheduler>((ctx, p) => {
if (string.IsNullOrEmpty(_options.Schedule) || _options.Mode != null && _options.Mode.In("init", "check"))
{
return(new NowScheduler(_options, ctx.Resolve <ISchemaHelper>(), ctx.Resolve <IPipelineLogger>()));
}
if (_options.Schedule == "internal")
{
var process = ProcessFactory.Create(_options.Arrangement, new Dictionary <string, string>());
if (process.Errors().Any())
{
Console.Error.WriteLine("In order for an internal schedule to work, the arrangement passed in must be valid!");
foreach (var error in process.Errors())
{
Console.Error.WriteLine(error);
}
Environment.Exit(1);
}
return(new QuartzCronSchedulerViaInternalSchedule(_options, process.Schedule, ctx.Resolve <IJobFactory>(), ctx.Resolve <ILoggerFactoryAdapter>()));
}
return(new QuartzCronSchedulerViaCommandLine(_options, ctx.Resolve <IJobFactory>(), ctx.Resolve <ILoggerFactoryAdapter>()));
}).As <IScheduler>();
}
protected override void Load(ContainerBuilder builder)
{
_options.Arrangement = _options.ArrangementWithMode();
builder.Register <IPipelineLogger>(c => new NLogPipelineLogger(SlugifyTransform.Slugify(_options.Arrangement))).As <IPipelineLogger>().SingleInstance();
builder.Register <IContext>(c => new PipelineContext(c.Resolve <IPipelineLogger>())).As <IContext>();
// for now scheduler
builder.Register(c => new RunTimeSchemaReader(c.Resolve <IContext>())).As <IRunTimeSchemaReader>();
builder.Register <ISchemaHelper>(ctx => new SchemaHelper(ctx.Resolve <IContext>(), ctx.Resolve <IRunTimeSchemaReader>())).As <ISchemaHelper>();
// for quartz scheduler
builder.RegisterType <QuartzJobFactory>().As <IJobFactory>().SingleInstance();
builder.Register <ILoggerFactoryAdapter>((ctx => new QuartzLogAdaptor(ctx.Resolve <IContext>(), Scheduler.Quartz.Utility.ConvertLevel(ctx.Resolve <IContext>().LogLevel), true, true, false, "o"))).As <ILoggerFactoryAdapter>();
builder.Register <IScheduler>((ctx, p) => {
if (string.IsNullOrEmpty(_options.Schedule) || _options.Mode != null && _options.Mode.In("init", "check"))
{
return(new NowScheduler(_options, ctx.Resolve <ISchemaHelper>()));
}
return(new QuartzCronScheduler(_options, ctx.Resolve <IJobFactory>(), ctx.Resolve <ILoggerFactoryAdapter>()));
}).As <IScheduler>();
}
protected override void Load(ContainerBuilder builder)
{
// This reader is used to load the initial configuration and nested resources for tfl actions, etc.
builder.RegisterType <FileReader>().Named <IReader>("file");
builder.RegisterType <WebReader>().Named <IReader>("web");
builder.Register <IReader>(ctx => new DefaultReader(
ctx.ResolveNamed <IReader>("file"),
new ReTryingReader(ctx.ResolveNamed <IReader>("web"), attempts: 3))
);
builder.Register((ctx, p) => {
var placeHolderStyle = "@()";
if (ctx.IsRegisteredWithName <string>("placeHolderStyle"))
{
placeHolderStyle = ctx.ResolveNamed <string>("placeHolderStyle");
}
var dependencies = new List <IDependency> {
ctx.Resolve <IReader>(),
new FormParameterModifier(new DateMathModifier()),
new EnvironmentModifier(new PlaceHolderReplacer(placeHolderStyle[0], placeHolderStyle[1], placeHolderStyle[2]))
};
if (ctx.IsRegisteredWithName <IDependency>("shorthand-t"))
{
dependencies.Add(ctx.ResolveNamed <IDependency>("shorthand-t"));
}
if (ctx.IsRegisteredWithName <IDependency>("shorthand-v"))
{
dependencies.Add(ctx.ResolveNamed <IDependency>("shorthand-v"));
}
var process = new Process(dependencies.ToArray());
switch (p.Count())
{
case 2:
process.Load(
p.Named <string>("cfg"),
p.Named <Dictionary <string, string> >("parameters")
);
break;
default:
process.Load(p.Named <string>("cfg"));
break;
}
if (process.Errors().Any())
{
return(process);
}
// this might be put into it's own type and injected (or not)
if (process.Entities.Count == 1)
{
var entity = process.Entities.First();
if (!entity.HasInput() && ctx.IsRegistered <ISchemaReader>())
{
var schemaReader = ctx.Resolve <ISchemaReader>(new TypedParameter(typeof(Process), process));
var schema = schemaReader.Read(entity);
var newEntity = schema.Entities.First();
foreach (var sf in newEntity.Fields.Where(f => f.Name == Constants.TflKey || f.Name == Constants.TflDeleted || f.Name == Constants.TflBatchId || f.Name == Constants.TflHashCode))
{
sf.Alias = newEntity.Name + sf.Name;
}
process.Entities.Clear();
process.Entities.Add(newEntity);
process.Connections.First(c => c.Name == newEntity.Connection).Delimiter = schema.Connection.Delimiter;
}
}
if (process.Output().Provider == Constants.DefaultSetting || process.Output().Provider == "internal")
{
try {
Console.WindowHeight = Console.WindowHeight + 1 - 1;
Console.Title = process.Name;
if (!System.Environment.CommandLine.Contains("TESTWINDOW") && !System.Environment.CommandLine.Contains("TESTPLATFORM"))
{
process.Output().Provider = "console";
}
} catch (Exception) {
// just a hack to determine if in console
}
}
// handling multiple entities with non-relational output
var originalOutput = process.Output().Clone();
originalOutput.Name = Constants.OriginalOutput;
originalOutput.Key = process.Name + originalOutput.Name;
if (!process.OutputIsRelational() && (process.Entities.Count > 1 || process.Buffer))
{
process.Output().Provider = process.InternalProvider;
var folder = Path.Combine(System.Environment.GetFolderPath(System.Environment.SpecialFolder.LocalApplicationData), Constants.ApplicationFolder);
var file = new FileInfo(Path.Combine(folder, SlugifyTransform.Slugify(process.Name) + "." + (process.InternalProvider == "sqlce" ? "sdf" : "sqlite3")));
var exists = file.Exists;
process.Output().File = file.FullName;
process.Output().RequestTimeout = process.InternalProvider == "sqlce" ? 0 : process.Output().RequestTimeout;
process.Flatten = process.InternalProvider == "sqlce";
process.Mode = exists ? process.Mode : "init";
process.Connections.Add(originalOutput);
if (!exists)
{
if (!Directory.Exists(file.DirectoryName))
{
Directory.CreateDirectory(folder);
}
}
}
return(process);
}).As <Process>().InstancePerDependency(); // because it has state, if you run it again, it's not so good
}
public void Start()
{
var builder = new ContainerBuilder();
builder.Register <IPipelineLogger>(c => new NLogPipelineLogger(SlugifyTransform.Slugify(_options.Arrangement))).As <IPipelineLogger>().SingleInstance();
builder.RegisterModule(new RootModule(_options.Shorthand));
builder.Register <IContext>(c => new PipelineContext(c.Resolve <IPipelineLogger>())).As <IContext>();
builder.Register(c => new NowExecutor(c.Resolve <IPipelineLogger>(), _options.Arrangement, _options.Shorthand, _options.Mode)).As <IRunTimeExecute>();
using (var scope = builder.Build().BeginLifetimeScope()) {
var context = scope.Resolve <IContext>();
var process = scope.Resolve <Configuration.Process>(
new NamedParameter("cfg", _options.Arrangement)
);
foreach (var warning in process.Warnings())
{
context.Debug(() => warning);
}
if (process.Errors().Any())
{
foreach (var error in process.Errors())
{
context.Error(error);
}
context.Error("The configuration errors must be fixed before this job will run.");
context.Logger.Clear();
return;
}
if (process.Entities.Any(e => process.Connections.First(c => c.Name == e.Connection).Provider != "internal" && !e.Fields.Any(f => f.Input)))
{
context.Debug(() => "Detecting schema...");
if (_schemaHelper.Help(process))
{
if (process.Errors().Any())
{
foreach (var error in process.Errors())
{
context.Error(error);
}
context.Error("The configuration errors must be fixed before this job will run.");
context.Logger.Clear();
return;
}
}
}
if (_options.Mode != null && _options.Mode.ToLower() == "check")
{
SimplifyForOutput(process);
Console.WriteLine(process.Serialize());
return;
}
if (_options.Mode != "default")
{
process.Mode = _options.Mode;
}
scope.Resolve <IRunTimeExecute>().Execute(process);
}
}
public void Start()
{
var builder = new ContainerBuilder();
builder.Register <IPipelineLogger>(c => new NLogPipelineLogger(SlugifyTransform.Slugify(_options.Arrangement))).As <IPipelineLogger>().SingleInstance();
builder.RegisterModule(new RootModule(_options.Shorthand));
builder.Register <IContext>(c => new PipelineContext(c.Resolve <IPipelineLogger>())).As <IContext>();
builder.Register(c => new RunTimeExecutor(_options.Arrangement, _options.Mode, _options.Format)).As <IRunTimeExecute>();
using (var scope = builder.Build().BeginLifetimeScope()) {
var context = scope.Resolve <IContext>();
var process = scope.Resolve <Configuration.Process>(
new NamedParameter("cfg", _options.Arrangement)
);
foreach (var warning in process.Warnings())
{
context.Warn(warning);
}
if (process.Errors().Any())
{
foreach (var error in process.Errors())
{
context.Error(error);
}
context.Error("The configuration errors must be fixed before this job will run.");
return;
}
if (process.Entities.Any(e => !e.Fields.Any(f => f.Input)))
{
context.Debug(() => "Detecting schema...");
if (_schemaHelper.Help(process))
{
if (process.Errors().Any())
{
foreach (var error in process.Errors())
{
context.Error(error);
}
context.Error("The configuration errors must be fixed before this job will run.");
return;
}
}
}
if (_options.Mode != null && _options.Mode.ToLower() == "check")
{
process.Star = string.Empty;
foreach (var connection in process.Connections)
{
connection.Delimiters.Clear();
}
foreach (var entity in process.Entities)
{
entity.CalculateHashCode = true;
if (entity.Name == entity.Alias)
{
entity.Alias = null;
}
entity.Fields.RemoveAll(f => f.System);
}
foreach (var field in process.GetAllFields().Where(f => !string.IsNullOrEmpty(f.T)))
{
field.T = string.Empty;
}
foreach (var field in process.GetAllFields())
{
if (field.Name == field.Alias)
{
field.Alias = null;
}
if (field.Name == field.Label)
{
field.Label = string.Empty;
}
field.SortField = string.Empty;
field.Sortable = Constants.DefaultSetting;
}
Console.WriteLine(process.Serialize());
return;
}
scope.Resolve <IRunTimeExecute>().Execute(process);
}
}
public void Execute(Process process)
{
process.Mode = _mode;
// Since we're in a Console app, and honor output format
if (process.Output().Provider.In("internal", "console"))
{
process.Output().Provider = "console";
process.Output().Format = _format;
}
var logger = new NLogPipelineLogger(SlugifyTransform.Slugify(_cfg));
if (!string.IsNullOrEmpty(process.MaxMemory))
{
var context = new PipelineContext(logger, process);
var timer = new Stopwatch();
timer.Start();
GC.Collect();
GC.WaitForPendingFinalizers();
timer.Stop();
context.Info($"Collected free memory. Time taken: {timer.Elapsed}.");
var currentBytes = System.Diagnostics.Process.GetCurrentProcess().WorkingSet64.Bytes();
var maxMemory = ByteSize.Parse(process.MaxMemory);
if (maxMemory.CompareTo(currentBytes) < 0)
{
context.Error($"Process exceeded {maxMemory.Megabytes.ToString("#.0")} Mb. Current memory is {currentBytes.Megabytes.ToString("#.0")} Mb!");
Environment.Exit(1);
}
else
{
context.Info($"The process is using {currentBytes.Megabytes.ToString("#.0")} Mb of it's max {maxMemory.Megabytes.ToString("#.0")} Mb allowed.");
}
}
var builder = new ContainerBuilder();
builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance();
builder.RegisterCallback(new RootModule(process.Shorthand).Configure);
builder.RegisterCallback(new ContextModule(process).Configure);
// providers
builder.RegisterCallback(new AdoModule(process).Configure);
builder.RegisterCallback(new LuceneModule(process).Configure);
builder.RegisterCallback(new SolrModule(process).Configure);
builder.RegisterCallback(new ElasticModule(process).Configure);
builder.RegisterCallback(new InternalModule(process).Configure);
builder.RegisterCallback(new FileModule(process).Configure);
builder.RegisterCallback(new FolderModule(process).Configure);
builder.RegisterCallback(new DirectoryModule(process).Configure);
builder.RegisterCallback(new ExcelModule(process).Configure);
builder.RegisterCallback(new WebModule(process).Configure);
builder.RegisterCallback(new MapModule(process).Configure);
builder.RegisterCallback(new TemplateModule(process).Configure);
builder.RegisterCallback(new ActionModule(process).Configure);
builder.RegisterCallback(new EntityPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessControlModule(process).Configure);
using (var scope = builder.Build().BeginLifetimeScope()) {
try {
scope.Resolve <IProcessController>().Execute();
} catch (Exception ex) {
scope.Resolve <IContext>().Error(ex.Message);
}
}
}
