public static bool TryCreate(Options options, Dictionary <string, string> parameters, out Process process)
{
var logger = new NLogPipelineLogger(SlugifyTransform.Slugify(options.Arrangement));
using (var scope = ConfigurationContainer.Create(options.Arrangement, logger, parameters, options.PlaceHolderStyle)) {
process = scope.Resolve <Process>();
var context = new PipelineContext(logger, process);
foreach (var warning in process.Warnings())
{
context.Warn(warning);
}
if (process.Errors().Any())
{
foreach (var error in process.Errors())
{
context.Error(error);
}
context.Error("The configuration errors must be fixed before this job will run.");
}
else
{
process.Preserve = true;
}
}
return(process != null && process.Errors().Length == 0);
}
static void Main(string[] args) {
var context = new PipelineContext(new ConsoleLogger(Logging.LogLevel.Debug), new Configuration.Process() { Name = "Command" });
if (args == null || args.Length == 0) {
context.Error("Please pass in a configuration.");
System.Environment.Exit(1);
}
var builder = new ContainerBuilder();
builder.RegisterModule(new ConfigurationModule(args[0], "Shorthand.xml"));
var cfg = builder.Build();
var root = cfg.Resolve<Root>();
if (root.Warnings().Any()) {
foreach (var warning in root.Warnings()) {
context.Warn(warning);
}
}
if (root.Errors().Any()) {
foreach (var error in root.Errors()) {
context.Error(error);
}
System.Environment.Exit(1);
}
context.Info("Configuration is Ok");
cfg.Dispose();
// register pipeline
builder = new ContainerBuilder();
builder.Register<IPipelineLogger>(ctx => new ConsoleLogger(LogLevel.Info)).SingleInstance();
builder.RegisterModule(new MapModule(root));
builder.RegisterModule(new TemplateModule(root));
builder.RegisterModule(new ActionModule(root));
builder.RegisterModule(new EntityControlModule(root));
builder.RegisterModule(new EntityInputModule(root));
builder.RegisterModule(new EntityOutputModule(root));
builder.RegisterModule(new EntityMasterUpdateModule(root));
builder.RegisterModule(new EntityDeleteModule(root));
builder.RegisterModule(new EntityPipelineModule(root));
builder.RegisterModule(new ProcessPipelineModule(root));
builder.RegisterModule(new ProcessControlModule(root));
using (var c = builder.Build().BeginLifetimeScope()) {
// resolve, run, and release
var container = c;
foreach (var controller in root.Processes.Select(process => container.ResolveNamed<IProcessController>(process.Key))) {
controller.PreExecute();
controller.Execute();
controller.PostExecute();
}
}
}
public void Start()
{
using (var outer = ConfigurationContainer.Create(_options.Arrangement, _logger, null, _options.PlaceHolderStyle)) {
var process = outer.Resolve <Configuration.Process>();
var log = new PipelineContext(_logger, process);
if (process.Entities.Any(e => process.Connections.First(c => c.Name == e.Connection).Provider != Constants.DefaultSetting && !e.Fields.Any(f => f.Input)))
{
log.Debug(() => "Detecting schema...");
if (_schemaHelper.Help(process))
{
process.Check();
}
}
if (_options.Mode != null && _options.Mode.ToLower() == "check")
{
SimplifyForOutput(process);
Console.WriteLine(process.Serialize());
return;
}
if (_options.Mode != "default")
{
process.Mode = _options.Mode;
}
foreach (var warning in process.Warnings())
{
log.Debug(() => warning);
}
if (process.Errors().Any())
{
foreach (var error in process.Errors())
{
log.Error(error);
}
log.Error("The configuration errors must be fixed before this job will run.");
log.Logger.Clear();
return;
}
using (var inner = DefaultContainer.Create(process, _logger, _options.PlaceHolderStyle)) {
inner.Resolve <IProcessController>().Execute();
}
}
}
public string Render()
{
var p = new Dictionary <string, string>();
var l = new Cfg.Net.Loggers.MemoryLogger();
// get template
_context.Debug(() => $"Reading {_template.File}");
var templateContent = _templateReader.Read(_template.File, p, l);
if (l.Errors().Any())
{
foreach (var error in l.Errors())
{
_context.Error(error);
}
return(string.Empty);
}
try {
_context.Debug(() => $"Compiling {_template.Name}.");
var context = new VelocityContext();
context.Put("Process", _context.Process);
foreach (var parameter in _template.Parameters)
{
context.Put(parameter.Name, Constants.ConversionMap[parameter.Type](parameter.Value));
}
if (p.Any())
{
foreach (var parameter in p)
{
context.Put(parameter.Key, parameter.Value);
}
}
var sb = new StringBuilder();
using (var sw = new StringWriter(sb)) {
NVelocity.App.Velocity.Evaluate(context, sw, string.Empty, templateContent);
sw.Flush();
}
return(sb.ToString());
} catch (Exception ex) {
_context.Error($"Error parsing template {_template.Name}.");
_context.Error(ex, ex.Message.Replace("{", "{{").Replace("}", "}}"));
return(string.Empty);
}
}
private static IAction SwitchAction(IComponentContext ctx, Process process, Configuration.Action action)
{
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process);
switch (action.Type)
{
case "log":
return(new LogAction(context, action));
case "wait":
case "sleep":
return(new WaitAction(action));
case "tfl":
var cfg = string.IsNullOrEmpty(action.Url) ? action.File : action.Url;
if (string.IsNullOrEmpty(cfg) && !string.IsNullOrEmpty(action.Body))
{
cfg = action.Body;
}
return(new PipelineAction(context, action, cfg));
case "exit":
return(new ExitAction(context, action));
default:
context.Error("{0} action is not registered.", action.Type);
return(new NullAction());
}
}
private static void CheckMemory(Process process, IPipelineLogger logger)
{
if (string.IsNullOrEmpty(process.MaxMemory))
{
return;
}
var context = new PipelineContext(logger, process);
GC.Collect();
GC.WaitForPendingFinalizers();
var currentBytes = System.Diagnostics.Process.GetCurrentProcess().WorkingSet64.Bytes();
var maxMemory = ByteSize.Parse(process.MaxMemory);
if (maxMemory.CompareTo(currentBytes) < 0)
{
context.Error(
$"Process exceeded {maxMemory.Megabytes.ToString("#.0")} Mb. Current memory is {currentBytes.Megabytes.ToString("#.0")} Mb!");
Environment.Exit(1);
}
else
{
context.Info(
$"The process is using {currentBytes.Megabytes.ToString("#.0")} Mb of it's max {maxMemory.Megabytes.ToString("#.0")} Mb allowed.");
}
}
private IAction SwitchAction(IComponentContext ctx, Action action)
{
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), _process);
switch (action.Type)
{
case "copy":
return(action.InTemplate ? (IAction)
new ContentToFileAction(context, action) :
new FileCopyAction(context, action));
case "move":
return(new FileMoveAction(context, action));
case "archive":
return(new FileArchiveAction(context, action));
case "replace":
return(new FileReplaceAction(context, action));
case "open":
return(new OpenAction(action));
default:
context.Error($"Attempting to register unsupported file sytem action: {action.Type}");
return(new NullAction());
}
}
public string Render()
{
var p = new Dictionary <string, string>();
var l = new Cfg.Net.Loggers.MemoryLogger();
// get template
_context.Debug(() => $"Reading {_template.File}");
var templateContent = _templateReader.Read(_template.File, p, l);
if (l.Errors().Any())
{
foreach (var error in l.Errors())
{
_context.Error(error);
}
return(string.Empty);
}
// get parameters (other than process)
var parameters = new ExpandoObject();
foreach (var parameter in _template.Parameters)
{
((IDictionary <string, object>)parameters).Add(parameter.Name, parameter.Value);
}
if (p.Any())
{
foreach (var parameter in p)
{
((IDictionary <string, object>)parameters)[parameter.Key] = parameter.Value;
}
}
try {
_context.Debug(() => $"Compiling {_template.Name}.");
return(_service.RunCompile(templateContent, _template.Name, null, new {
_context.Process,
Parameters = parameters
}));
} catch (Exception ex) {
_context.Error($"Error parsing template {_template.Name}.");
_context.Error(ex, ex.Message.Replace("{", "{{").Replace("}", "}}"));
return(string.Empty);
}
}
private static IAction SwitchAction(IComponentContext ctx, Process process, Configuration.Action action)
{
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process);
switch (action.Type)
{
case "log":
return(new LogAction(context, action));
case "wait":
case "sleep":
return(new WaitAction(action));
case "tfl":
var cfg = string.IsNullOrEmpty(action.Url) ? action.File : action.Url;
if (string.IsNullOrEmpty(cfg) && !string.IsNullOrEmpty(action.Body))
{
cfg = action.Body;
}
var root = ctx.Resolve <Process>(new NamedParameter("cfg", cfg));
foreach (var warning in root.Warnings())
{
context.Warn(warning);
}
if (root.Errors().Any())
{
context.Error($"TFL Pipeline Action '{cfg.Left(30)}'... has errors!");
foreach (var error in root.Errors())
{
context.Error(error);
}
return(new NullAction());
}
return(new PipelineAction(DefaultContainer.Create(root, ctx.Resolve <IPipelineLogger>(), action.PlaceHolderStyle)));
case "exit":
return(new ExitAction(context, action));
default:
context.Error("{0} action is not registered.", action.Type);
return(new NullAction());
}
}
private static IAction SwitchAction(IComponentContext ctx, Process process, Action action)
{
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process);
switch (action.Type)
{
case "log":
return(new LogAction(context, action));
default:
context.Error("{0} action is not registered.", action.Type);
return(new NullAction());
}
}
private static IAction SwitchAction(IComponentContext ctx, Process process, Action action) {
var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process);
switch (action.Name) {
case "copy":
return action.InTemplate ? (IAction)
new ContentToFileAction(context, action) :
new FileToFileAction(context, action);
case "web":
return new WebAction(context, action);
default:
context.Error("{0} action is not registered.", action.Name);
return new NullAction();
}
}
private IAction SwitchAction(IComponentContext ctx, Action action)
{
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), _process);
switch (action.Type)
{
case "print":
return(new PrintAction(action));
default:
context.Error($"Attempting to register unsupported file sytem action: {action.Type}");
return(new NullAction());
}
}
private static void CheckMemory(Process process, IPipelineLogger logger)
{
if (string.IsNullOrEmpty(process.MaxMemory))
{
return;
}
var context = new PipelineContext(logger, process);
GC.Collect();
GC.WaitForPendingFinalizers();
var bytes = System.Diagnostics.Process.GetCurrentProcess().WorkingSet64.Bytes();
if (bytes.Megabytes > MaxBytes.Megabytes)
{
if (MaxBytes.Megabytes > 0.0)
{
context.Warn($"Process memory has increased from {MaxBytes.Megabytes:#.0} to {bytes.Megabytes:#.0}.");
}
MaxBytes = bytes;
}
var maxMemory = ByteSize.Parse(process.MaxMemory);
if (maxMemory.CompareTo(bytes) < 0)
{
context.Error($"Process exceeded {maxMemory.Megabytes:#.0} Mb. Current memory is {bytes.Megabytes:#.0} Mb!");
context.Error($"Process is exiting after running {RunCount} time{RunCount.Plural()}.");
Environment.Exit(1);
}
else
{
context.Info($"Process is using {bytes.Megabytes:#.0} Mb of it's max {maxMemory.Megabytes:#.0} Mb allowed.");
context.Info($"Process has run {RunCount} time{RunCount.Plural()}.");
}
}
public ActionResponse Execute()
{
var response = new ActionResponse {
Action = _action
};
var to = new FileInfo(_action.To);
if (string.IsNullOrEmpty(_action.Body))
{
_context.Warn("Nothing to write to {0}", to.Name);
}
else
{
_context.Info("Writing content to {0}", to.Name);
try {
System.IO.File.WriteAllText(to.FullName, _action.Body);
} catch (Exception ex) {
_context.Error($"Failed to run {_action.Type} action.");
_context.Error(ex.Message);
response.Code = 500;
}
}
return(response);
}
private static IAction SwitchAction(IComponentContext ctx, Process process, Action action)
{
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process);
switch (action.Name)
{
case "copy":
return(action.InTemplate ? (IAction)
new ContentToFileAction(context, action) :
new FileToFileAction(context, action));
case "web":
return(new WebAction(context, action));
default:
context.Error("{0} action is not registered.", action.Name);
return(new NullAction());
}
}
public void Error(string message, params object[] args)
{
_context.Error(message, args);
}
private static IAction SwitchAction(IComponentContext ctx, Process process, Action action)
{
Connection connection;
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process);
switch (action.Type)
{
case "copy":
return(action.InTemplate ? (IAction)
new ContentToFileAction(context, action) :
new FileCopyAction(context, action));
case "move":
return(new FileMoveAction(context, action));
case "replace":
return(new FileReplaceAction(context, action));
case "log":
return(new LogAction(context, action));
case "web":
return(new WebAction(context, action));
case "form-commands":
connection = process.Connections.First(c => c.Name == action.Connection);
return(new AdoEntityFormCommands(context, action, ctx.ResolveNamed <IConnectionFactory>(connection.Key)));
case "humanize-labels":
return(new HumanizeLabels(context, action));
case "wait":
case "sleep":
return(new WaitAction(action));
case "tfl":
var cfg = string.IsNullOrEmpty(action.Url) ? action.File : action.Url;
if (string.IsNullOrEmpty(cfg) && !string.IsNullOrEmpty(action.Body))
{
cfg = action.Body;
}
var root = ctx.Resolve <Process>(new NamedParameter("cfg", cfg));
foreach (var warning in root.Warnings())
{
context.Warn(warning);
}
if (root.Errors().Any())
{
context.Error(string.Format("TFL Pipeline Action '{0}' has errors!", cfg.Left(15) + "..." + cfg.Right(15)));
foreach (var error in root.Errors())
{
context.Error(error);
}
return(new NullAction());
}
var builder = new ContainerBuilder();
// Register Orchard CMS Stuff
builder.RegisterInstance(ctx.Resolve <IAppDataFolder>()).As <IAppDataFolder>();
builder.RegisterInstance(ctx.Resolve <ITemplateProcessor>()).As <ITemplateProcessor>();
builder.RegisterInstance(ctx.Resolve <INotifier>()).As <INotifier>();
builder.RegisterInstance(context.Logger).As <IPipelineLogger>();
builder.RegisterCallback(new RootModule().Configure);
builder.RegisterCallback(new ContextModule(root).Configure);
// providers
builder.RegisterCallback(new AdoModule(root).Configure);
builder.RegisterCallback(new SolrModule(root).Configure);
builder.RegisterCallback(new ElasticModule(root).Configure);
builder.RegisterCallback(new InternalModule(root).Configure);
builder.RegisterCallback(new FileModule(ctx.Resolve <IAppDataFolder>()).Configure);
builder.RegisterCallback(new ExcelModule().Configure);
builder.RegisterCallback(new GeoJsonModule().Configure);
builder.RegisterCallback(new KmlModule().Configure);
builder.RegisterCallback(new TransformModule(root).Configure);
builder.RegisterCallback(new ValidateModule(root).Configure);
builder.RegisterCallback(new MapModule(root).Configure);
builder.RegisterCallback(new ActionModule(root).Configure);
builder.RegisterCallback(new EntityPipelineModule(root).Configure);
builder.RegisterCallback(new ProcessPipelineModule(root).Configure);
builder.RegisterCallback(new ProcessControlModule(root).Configure);
return(new PipelineAction(builder.Build(), root));
case "run":
connection = process.Connections.First(c => c.Name == action.Connection);
switch (connection.Provider)
{
case "mysql":
case "postgresql":
case "sqlite":
case "sqlserver":
return(new AdoRunAction(context, action, ctx.ResolveNamed <IConnectionFactory>(connection.Key), new DefaultReader(new FileReader(), new WebReader())));
default:
context.Error("{0} provider is not registered for run action.", connection.Provider);
return(new NullAction());
}
case "open":
return(new OpenAction(action));
default:
context.Error("{0} action is not registered.", action.Type);
return(new NullAction());
}
}
public void Execute(Process process)
{
process.Mode = _mode;
// Since we're in a Console app, and honor output format
if (process.Output().Provider.In("internal", "console"))
{
process.Output().Provider = "console";
process.Output().Format = _format;
}
var logger = new NLogPipelineLogger(SlugifyTransform.Slugify(_cfg));
if (!string.IsNullOrEmpty(process.MaxMemory))
{
var context = new PipelineContext(logger, process);
var timer = new Stopwatch();
timer.Start();
GC.Collect();
GC.WaitForPendingFinalizers();
timer.Stop();
context.Info($"Collected free memory. Time taken: {timer.Elapsed}.");
var currentBytes = System.Diagnostics.Process.GetCurrentProcess().WorkingSet64.Bytes();
var maxMemory = ByteSize.Parse(process.MaxMemory);
if (maxMemory.CompareTo(currentBytes) < 0)
{
context.Error($"Process exceeded {maxMemory.Megabytes.ToString("#.0")} Mb. Current memory is {currentBytes.Megabytes.ToString("#.0")} Mb!");
Environment.Exit(1);
}
else
{
context.Info($"The process is using {currentBytes.Megabytes.ToString("#.0")} Mb of it's max {maxMemory.Megabytes.ToString("#.0")} Mb allowed.");
}
}
var builder = new ContainerBuilder();
builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance();
builder.RegisterCallback(new RootModule(process.Shorthand).Configure);
builder.RegisterCallback(new ContextModule(process).Configure);
// providers
builder.RegisterCallback(new AdoModule(process).Configure);
builder.RegisterCallback(new LuceneModule(process).Configure);
builder.RegisterCallback(new SolrModule(process).Configure);
builder.RegisterCallback(new ElasticModule(process).Configure);
builder.RegisterCallback(new InternalModule(process).Configure);
builder.RegisterCallback(new FileModule(process).Configure);
builder.RegisterCallback(new FolderModule(process).Configure);
builder.RegisterCallback(new DirectoryModule(process).Configure);
builder.RegisterCallback(new ExcelModule(process).Configure);
builder.RegisterCallback(new WebModule(process).Configure);
builder.RegisterCallback(new MapModule(process).Configure);
builder.RegisterCallback(new TemplateModule(process).Configure);
builder.RegisterCallback(new ActionModule(process).Configure);
builder.RegisterCallback(new EntityPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessControlModule(process).Configure);
using (var scope = builder.Build().BeginLifetimeScope()) {
try {
scope.Resolve <IProcessController>().Execute();
} catch (Exception ex) {
scope.Resolve <IContext>().Error(ex.Message);
}
}
}
public ILifetimeScope CreateScope(string cfg, IPipelineLogger logger, IDictionary <string, string> parameters = null, string placeHolderStyle = "@[]")
{
if (placeHolderStyle == null || placeHolderStyle.Length != 3)
{
throw new ArgumentException("The placeHolderStyle parameter must be three characters (e.g. @[], or @())");
}
if (parameters == null)
{
parameters = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
}
var builder = new ContainerBuilder();
builder.Register((ctx) => logger).As <IPipelineLogger>();
builder.Register <IReader>(c => new DefaultReader(new FileReader(), new WebReader())).As <IReader>();
// register short-hand for t attribute
var transformModule = new TransformModule(new Process {
Name = "TransformShorthand"
}, _methods, _shortHand, logger);
foreach (var t in _transforms)
{
transformModule.AddTransform(t);
}
builder.RegisterModule(transformModule);
// register short-hand for v attribute
var validateModule = new ValidateModule(new Process {
Name = "ValidateShorthand"
}, _methods, _shortHand, logger);
foreach (var v in _validators)
{
validateModule.AddValidator(v);
}
builder.RegisterModule(validateModule);
// register the validator short hand
builder.Register((c, p) => _shortHand).Named <ShorthandRoot>(ValidateModule.FieldsName).InstancePerLifetimeScope();
builder.Register((c, p) => new ShorthandCustomizer(c.ResolveNamed <ShorthandRoot>(ValidateModule.FieldsName), new[] { "fields", "calculated-fields", "calculatedfields" }, "v", "validators", "method")).Named <IDependency>(ValidateModule.FieldsName).InstancePerLifetimeScope();
// register the transform short hand
builder.Register((c, p) => _shortHand).Named <ShorthandRoot>(TransformModule.FieldsName).InstancePerLifetimeScope();
builder.Register((c, p) => _shortHand).Named <ShorthandRoot>(TransformModule.ParametersName).InstancePerLifetimeScope();
builder.Register((c, p) => new ShorthandCustomizer(c.ResolveNamed <ShorthandRoot>(TransformModule.FieldsName), new[] { "fields", "calculated-fields", "calculatedfields" }, "t", "transforms", "method")).Named <IDependency>(TransformModule.FieldsName).InstancePerLifetimeScope();
builder.Register((c, p) => new ShorthandCustomizer(c.ResolveNamed <ShorthandRoot>(TransformModule.ParametersName), new[] { "parameters" }, "t", "transforms", "method")).Named <IDependency>(TransformModule.ParametersName).InstancePerLifetimeScope();
#if PLUGINS
// the shorthand registrations are stored in the builder's properties for plugins to add to
builder.Properties["ShortHand"] = _shortHand;
builder.Properties["Methods"] = _methods;
#endif
foreach (var module in _modules)
{
builder.RegisterModule(module);
}
builder.Register((c, p) => _methods).Named <HashSet <string> >("Methods").InstancePerLifetimeScope();
builder.Register((c, p) => _shortHand).As <ShorthandRoot>().InstancePerLifetimeScope();
builder.Register(ctx => {
var transformed = TransformParameters(ctx, cfg);
var dependancies = new List <IDependency>();
dependancies.Add(ctx.Resolve <IReader>());
dependancies.AddRange(_dependencies);
dependancies.Add(new ParameterModifier(new PlaceHolderReplacer(placeHolderStyle[0], placeHolderStyle[1], placeHolderStyle[2]), "parameters", "name", "value"));
dependancies.Add(ctx.ResolveNamed <IDependency>(TransformModule.FieldsName));
dependancies.Add(ctx.ResolveNamed <IDependency>(TransformModule.ParametersName));
dependancies.Add(ctx.ResolveNamed <IDependency>(ValidateModule.FieldsName));
var process = new Process(transformed ?? cfg, parameters, dependancies.ToArray());
if (process.Errors().Any())
{
var c = new PipelineContext(logger, new Process()
{
Name = "Errors"
});
c.Error("The configuration has errors.");
foreach (var error in process.Errors())
{
c.Error(error);
}
}
return(process);
}).As <Process>().InstancePerDependency();
return(builder.Build().BeginLifetimeScope());
}
private static string TransformParameters(IComponentContext ctx, string cfg)
{
var preProcess = new ConfigurationFacade.Process(
cfg,
new Dictionary <string, string>(),
new List <IDependency> {
ctx.Resolve <IReader>(),
new DateMathModifier(),
new ParameterModifier(new NullPlaceHolderReplacer()),
ctx.ResolveNamed <IDependency>(TransformModule.ParametersName)
}.ToArray());
if (!preProcess.Parameters.Any(pr => pr.Transforms.Any()))
{
return(null);
}
var fields = preProcess.Parameters.Select(pr => new Field {
Name = pr.Name,
Alias = pr.Name,
Default = pr.Value,
Type = pr.Type,
Transforms = pr.Transforms.Select(o => o.ToOperation()).ToList()
}).ToList();
var len = fields.Count;
var entity = new Entity {
Name = "Parameters", Alias = "Parameters", Fields = fields
};
var mini = new Process {
Name = "ParameterTransform",
ReadOnly = true,
Entities = new List <Entity> {
entity
},
Maps = preProcess.Maps.Select(m => m.ToMap()).ToList(), // for map transforms
Scripts = preProcess.Scripts.Select(s => s.ToScript()).ToList() // for transforms that use scripts (e.g. js)
};
mini.Load(); // very important to check after creating, as it runs validation and even modifies!
if (!mini.Errors().Any())
{
// modification in Load() do not make it out to local variables so overwrite them
fields = mini.Entities.First().Fields;
entity = mini.Entities.First();
var c = new PipelineContext(ctx.Resolve <IPipelineLogger>(), mini, entity);
var transforms = new List <ITransform> {
new Transforms.System.DefaultTransform(c, fields)
};
transforms.AddRange(TransformFactory.GetTransforms(ctx, c, fields));
// make an input out of the parameters
var input = new List <IRow>();
var row = new MasterRow(len);
for (var i = 0; i < len; i++)
{
row[fields[i]] = preProcess.Parameters[i].Value;
}
input.Add(row);
var output = transforms.Aggregate(input.AsEnumerable(), (rows, t) => t.Operate(rows)).ToList().First();
for (var i = 0; i < len; i++)
{
var parameter = preProcess.Parameters[i];
parameter.Value = output[fields[i]].ToString();
parameter.T = string.Empty;
parameter.Transforms.Clear();
}
return(preProcess.Serialize());
}
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), mini, entity);
foreach (var error in mini.Errors())
{
context.Error(error);
}
return(null);
}
private static IAction SwitchAction(IComponentContext ctx, Process process, Action action)
{
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process);
switch (action.Type)
{
case "copy":
return(action.InTemplate ? (IAction)
new ContentToFileAction(context, action) :
new FileCopyAction(context, action));
case "move":
return(new FileMoveAction(context, action));
case "replace":
return(new FileReplaceAction(context, action));
case "print":
return(new PrintAction(action));
case "log":
return(new LogAction(context, action));
case "web":
return(new WebAction(context, action));
case "wait":
case "sleep":
return(new WaitAction(action));
case "tfl":
var cfg = string.IsNullOrEmpty(action.Url) ? action.File : action.Url;
if (string.IsNullOrEmpty(cfg) && !string.IsNullOrEmpty(action.Body))
{
cfg = action.Body;
}
var root = ctx.Resolve <Process>(new NamedParameter("cfg", cfg));
foreach (var warning in root.Warnings())
{
context.Warn(warning);
}
if (root.Errors().Any())
{
context.Error($"TFL Pipeline Action '{cfg.Left(15) + "..." + cfg.Right(15)}' has errors!");
foreach (var error in root.Errors())
{
context.Error(error);
}
return(new NullAction());
}
return(new PipelineAction(context, root));
case "run":
var connection = process.Connections.First(c => c.Name == action.Connection);
switch (connection.Provider)
{
case "mysql":
case "postgresql":
case "sqlite":
case "sqlserver":
return(new AdoRunAction(context, action, ctx.ResolveNamed <IConnectionFactory>(connection.Key), new DefaultReader(new FileReader(), new WebReader())));
default:
context.Error("{0} provider is not registered for run action.", connection.Provider);
return(new NullAction());
}
case "open":
return(new OpenAction(action));
default:
context.Error("{0} action is not registered.", action.Type);
return(new NullAction());
}
}
private static IAction SwitchAction(IComponentContext ctx, Process process, Action action)
{
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process);
switch (action.Type)
{
case "copy":
return(action.InTemplate ? (IAction)
new ContentToFileAction(context, action) :
new FileToFileAction(context, action));
case "web":
return(new WebAction(context, action));
case "tfl":
var cfg = string.IsNullOrEmpty(action.Url) ? action.File : action.Url;
if (string.IsNullOrEmpty(cfg) && !string.IsNullOrEmpty(action.Content))
{
cfg = action.Content;
}
var root = ctx.Resolve <Process>(new NamedParameter("cfg", cfg));
foreach (var warning in root.Warnings())
{
context.Warn(warning);
}
if (root.Errors().Any())
{
context.Error($"TFL Pipeline Action '{cfg.Left(15) + "..." + cfg.Right(15)}' has errors!");
foreach (var error in root.Errors())
{
context.Error(error);
}
return(new NullAction());
}
var builder = new ContainerBuilder();
builder.RegisterInstance(context.Logger);
builder.RegisterCallback(new RootModule(action.Shorthand).Configure);
builder.RegisterCallback(new ContextModule(root).Configure);
// providers
builder.RegisterCallback(new AdoModule(root).Configure);
builder.RegisterCallback(new LuceneModule(root).Configure);
builder.RegisterCallback(new SolrModule(root).Configure);
builder.RegisterCallback(new ElasticModule(root).Configure);
builder.RegisterCallback(new InternalModule(root).Configure);
builder.RegisterCallback(new FileModule(root).Configure);
builder.RegisterCallback(new FolderModule(root).Configure);
builder.RegisterCallback(new DirectoryModule(process).Configure);
builder.RegisterCallback(new ExcelModule(root).Configure);
builder.RegisterCallback(new WebModule(root).Configure);
builder.RegisterCallback(new MapModule(root).Configure);
builder.RegisterCallback(new TemplateModule(root).Configure);
builder.RegisterCallback(new ActionModule(root).Configure);
builder.RegisterCallback(new EntityPipelineModule(root).Configure);
builder.RegisterCallback(new ProcessPipelineModule(root).Configure);
builder.RegisterCallback(new ProcessControlModule(root).Configure);
return(new PipelineAction(builder.Build(), root));
case "run":
var connection = process.Connections.First(c => c.Name == action.Connection);
switch (connection.Provider)
{
case "mysql":
case "postgresql":
case "sqlite":
case "sqlserver":
return(new AdoRunAction(action, ctx.ResolveNamed <IConnectionFactory>(connection.Key)));
default:
context.Error("{0} provider is not registered for run action.", connection.Provider);
return(new NullAction());
}
case "open":
return(new OpenAction(action));
default:
context.Error("{0} action is not registered.", action.Type);
return(new NullAction());
}
}
static void Main(string[] args)
{
var context = new PipelineContext(new ConsoleLogger(Logging.LogLevel.Debug), new Configuration.Process()
{
Name = "Command"
});
if (args == null || args.Length == 0)
{
context.Error("Please pass in a configuration.");
System.Environment.Exit(1);
}
var builder = new ContainerBuilder();
builder.RegisterModule(new ConfigurationModule(args[0], "Shorthand.xml"));
var cfg = builder.Build();
var root = cfg.Resolve <Root>();
if (root.Warnings().Any())
{
foreach (var warning in root.Warnings())
{
context.Warn(warning);
}
}
if (root.Errors().Any())
{
foreach (var error in root.Errors())
{
context.Error(error);
}
System.Environment.Exit(1);
}
context.Info("Configuration is Ok");
cfg.Dispose();
// register pipeline
builder = new ContainerBuilder();
builder.Register <IPipelineLogger>(ctx => new ConsoleLogger(LogLevel.Info)).SingleInstance();
builder.RegisterModule(new MapModule(root));
builder.RegisterModule(new TemplateModule(root));
builder.RegisterModule(new ActionModule(root));
builder.RegisterModule(new EntityControlModule(root));
builder.RegisterModule(new EntityInputModule(root));
builder.RegisterModule(new EntityOutputModule(root));
builder.RegisterModule(new EntityMasterUpdateModule(root));
builder.RegisterModule(new EntityDeleteModule(root));
builder.RegisterModule(new EntityPipelineModule(root));
builder.RegisterModule(new ProcessPipelineModule(root));
builder.RegisterModule(new ProcessControlModule(root));
using (var c = builder.Build().BeginLifetimeScope()) {
// resolve, run, and release
var container = c;
foreach (var controller in root.Processes.Select(process => container.ResolveNamed <IProcessController>(process.Key)))
{
controller.PreExecute();
controller.Execute();
controller.PostExecute();
}
}
}
static void Run(RunOptions options)
{
var logger = new ConsoleLogger(options.LogLevel);
var operations = new List <Autofac.Core.IModule> {
new JintTransformModule(),
new RazorTransformModule(),
new FluidTransformModule(),
new HumanizeModule(),
new LambdaParserModule()
};
// todo: geoCode, etc
using (var outer = new ConfigurationContainer(operations.ToArray()).CreateScope(options.ArrangementWithMode(), logger, options.GetParameters())) {
var process = outer.Resolve <Process>();
if (options.Mode != "default" && options.Mode != process.Mode)
{
process.Mode = options.Mode;
process.Load();
}
if (process.Errors().Any())
{
var context = new PipelineContext(logger, process);
context.Error("The configuration has errors.");
foreach (var error in process.Errors())
{
context.Error(error);
}
Environment.Exit(1);
}
var providers = new List <Autofac.Core.IModule> {
new ConsoleProviderModule()
};
var output = process.GetOutputConnection();
if (output == null || output.Provider == "internal" || output.Provider == "console")
{
logger.SuppressConsole();
if (options.Format == "csv")
{
output.Provider = "file"; // delimited file
output.Delimiter = ",";
output.Stream = true;
output.Synchronous = true; // got odd results when using Async methods
output.File = "dummy.csv";
providers.Add(new CsvHelperProviderModule(Console.OpenStandardOutput()));
}
else
{
output.Provider = "json";
output.Stream = true;
output.Format = "json";
output.File = "dummy.json";
providers.Add(new JsonProviderModule(Console.OpenStandardOutput()));
}
}
else
{
providers.Add(new CsvHelperProviderModule());
providers.Add(new JsonProviderModule());
}
// PROVIDERS
providers.Add(new AdoProviderModule());
providers.Add(new BogusModule());
providers.Add(new SqliteModule());
providers.Add(new SqlServerModule());
providers.Add(new PostgreSqlModule());
providers.Add(new MySqlModule());
providers.Add(new ElasticsearchModule());
providers.Add(new RazorProviderModule());
providers.Add(new AwsCloudWatchProviderModule());
providers.Add(new AmazonConnectProviderModule());
providers.Add(new MailModule());
// solr
var modules = providers.Union(operations).ToArray();
if (options.Mode.ToLower() == "schema")
{
using (var inner = new Container(modules).CreateScope(process, logger)) {
process = new SchemaService(inner).GetProcess(process);
process.Connections.Clear();
Console.WriteLine(process.Serialize());
Environment.Exit(0);
}
}
else if (process.Entities.Count == 1 && !process.Entities[0].Fields.Any(f => f.Input))
{
using (var inner = new Container(modules).CreateScope(process, logger)) {
if (new SchemaService(inner).Help(process))
{
process.Load();
}
else
{
Console.Error.WriteLine($"Unable to detect fields in {process.Entities[0].Name}.");
Environment.Exit(1);
}
}
}
using (var inner = new Container(modules).CreateScope(process, logger)) {
inner.Resolve <IProcessController>().Execute();
}
}
}
