public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) {
builder.Register<IRead>(ctx => {
var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity);
var input = new InputContext(context, new Incrementer(context));
switch (input.Connection.Provider) {
case "internal":
context.Debug("Registering {0} provider", input.Connection.Provider);
return new DataSetEntityReader(input);
case "sqlserver":
if (input.Entity.ReadSize == 0) {
context.Debug("Registering {0} reader", input.Connection.Provider);
return new SqlInputReader(input, input.InputFields);
}
context.Debug("Registering {0} batch reader", input.Connection.Provider);
return new SqlInputBatchReader(
input,
new SqlInputReader(input, input.Entity.GetPrimaryKey())
);
default:
context.Warn("Registering null reader", input.Connection.Provider);
return new NullEntityReader();
}
}).Named<IRead>(entity.Key);
}
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity)
{
builder.Register <IRead>(ctx => {
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity);
var input = new InputContext(context, new Incrementer(context));
switch (input.Connection.Provider)
{
case "internal":
context.Debug("Registering {0} provider", input.Connection.Provider);
return(new DataSetEntityReader(input));
case "sqlserver":
if (input.Entity.ReadSize == 0)
{
context.Debug("Registering {0} reader", input.Connection.Provider);
return(new SqlInputReader(input, input.InputFields));
}
context.Debug("Registering {0} batch reader", input.Connection.Provider);
return(new SqlInputBatchReader(
input,
new SqlInputReader(input, input.Entity.GetPrimaryKey())
));
default:
context.Warn("Registering null reader", input.Connection.Provider);
return(new NullEntityReader());
}
}).Named <IRead>(entity.Key);
}
public string Render()
{
var p = new Dictionary <string, string>();
var l = new Cfg.Net.Loggers.MemoryLogger();
// get template
_context.Debug(() => $"Reading {_template.File}");
var templateContent = _templateReader.Read(_template.File, p, l);
if (l.Errors().Any())
{
foreach (var error in l.Errors())
{
_context.Error(error);
}
return(string.Empty);
}
try {
_context.Debug(() => $"Compiling {_template.Name}.");
var context = new VelocityContext();
context.Put("Process", _context.Process);
foreach (var parameter in _template.Parameters)
{
context.Put(parameter.Name, Constants.ConversionMap[parameter.Type](parameter.Value));
}
if (p.Any())
{
foreach (var parameter in p)
{
context.Put(parameter.Key, parameter.Value);
}
}
var sb = new StringBuilder();
using (var sw = new StringWriter(sb)) {
NVelocity.App.Velocity.Evaluate(context, sw, string.Empty, templateContent);
sw.Flush();
}
return(sb.ToString());
} catch (Exception ex) {
_context.Error($"Error parsing template {_template.Name}.");
_context.Error(ex, ex.Message.Replace("{", "{{").Replace("}", "}}"));
return(string.Empty);
}
}
public void Start()
{
using (var outer = ConfigurationContainer.Create(_options.Arrangement, _logger, null, _options.PlaceHolderStyle)) {
var process = outer.Resolve <Configuration.Process>();
var log = new PipelineContext(_logger, process);
if (process.Entities.Any(e => process.Connections.First(c => c.Name == e.Connection).Provider != Constants.DefaultSetting && !e.Fields.Any(f => f.Input)))
{
log.Debug(() => "Detecting schema...");
if (_schemaHelper.Help(process))
{
process.Check();
}
}
if (_options.Mode != null && _options.Mode.ToLower() == "check")
{
SimplifyForOutput(process);
Console.WriteLine(process.Serialize());
return;
}
if (_options.Mode != "default")
{
process.Mode = _options.Mode;
}
foreach (var warning in process.Warnings())
{
log.Debug(() => warning);
}
if (process.Errors().Any())
{
foreach (var error in process.Errors())
{
log.Error(error);
}
log.Error("The configuration errors must be fixed before this job will run.");
log.Logger.Clear();
return;
}
using (var inner = DefaultContainer.Create(process, _logger, _options.PlaceHolderStyle)) {
inner.Resolve <IProcessController>().Execute();
}
}
}
protected override void Load(ContainerBuilder builder)
{
if (_process == null)
{
return;
}
// razor moved to plugin, velocity too soon...
foreach (var t in _process.Templates.Where(t => t.Enabled && t.Engine != "razor"))
{
var template = t;
builder.Register <ITemplateEngine>(ctx => {
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), _process);
context.Debug(() => $"Registering {template.Engine} Engine for {t.Key}");
switch (template.Engine)
{
case "velocity":
return(new VelocityTemplateEngine(context, template, ctx.Resolve <IReader>()));
default:
return(new NullTemplateEngine());
}
}).Named <ITemplateEngine>(t.Key);
}
}
protected override void Load(ContainerBuilder builder)
{
new TransformBuilder(_process, builder, _methods, _shortHand, _transforms, _logger).Build();
#if PLUGINS
var context = new PipelineContext(_logger, _process);
builder.Properties["ShortHand"] = _shortHand;
builder.Properties["Methods"] = _methods;
builder.Properties["Process"] = _process;
var pluginsFolder = Path.Combine(AssemblyDirectory, "plugins");
if (Directory.Exists(pluginsFolder))
{
var assemblies = new List <Assembly>();
foreach (var file in Directory.GetFiles(pluginsFolder, "Transformalize.Transform.*.Autofac.dll", SearchOption.TopDirectoryOnly))
{
var info = new FileInfo(file);
var name = info.Name.ToLower().Split('.').FirstOrDefault(f => f != "dll" && f != "transformalize" && f != "transform" && f != "autofac");
context.Debug(() => $"Loading {name} transform(s)");
var assembly = Assembly.LoadFile(new FileInfo(file).FullName);
assemblies.Add(assembly);
}
if (assemblies.Any())
{
builder.RegisterAssemblyModules(assemblies.ToArray());
}
}
#endif
}
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) {
builder.Register<IEntityController>(ctx => {
var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity);
var output = new OutputContext(context, new Incrementer(context));
switch (output.Connection.Provider) {
case "sqlserver":
context.Debug("Registering sql server controller");
var initializer = process.Mode == "init" ? (IAction)new SqlEntityInitializer(output) : new NullInitializer();
return new SqlEntityController(output, initializer);
default:
context.Debug("Registering null controller");
return new NullEntityController();
}
}).Named<IEntityController>(entity.Key);
}
protected override void Load(ContainerBuilder builder)
{
var loadContext = new PipelineContext(_logger, _process);
builder.Properties["ShortHand"] = _shortHand;
builder.Properties["Methods"] = _methods;
builder.Properties["Process"] = _process;
// return true or false, validators
// new style
RegisterValidator(builder, (ctx, c) => new AnyValidator(c), new AnyValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new AllValidator(c), new AllValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new StartsWithValidator(c), new StartsWithValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new EndsWithValidator(c), new EndsWithValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new MapValidator(true, c), new MapValidator(inMap: true).GetSignatures());
RegisterValidator(builder, (ctx, c) => new MapValidator(false, c), new MapValidator(inMap: false).GetSignatures());
RegisterValidator(builder, (ctx, c) => new ContainsValidator(c), new ContainsValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new IsValidator(c), new IsValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new EqualsValidator(c), new EqualsValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new NotEqualValidator(c), new NotEqualValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new EmptyValidator(c), new EmptyValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new DefaultValidator(c), new DefaultValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new NumericValidator(c), new NumericValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new MatchValidator(c), new MatchValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new RequiredValidator(c), new RequiredValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new LengthValidator(c), new LengthValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new MinLengthValidator(c), new MinLengthValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new MaxLengthValidator(c), new MaxLengthValidator().GetSignatures());
RegisterValidator(builder, (ctx, c) => new CompareValidator("min", c), new CompareValidator("min").GetSignatures());
RegisterValidator(builder, (ctx, c) => new CompareValidator("max", c), new CompareValidator("max").GetSignatures());
RegisterValidator(builder, (ctx, c) => new RegularExpressionValidator("alphanum", "^[a-zA-Z0-9]*$", "must be alphanumeric", c), new RegularExpressionValidator("alphanum", "^[a-zA-Z0-9]*$", "must be alphanumeric").GetSignatures());
var pluginsFolder = Path.Combine(AssemblyDirectory, "plugins");
if (Directory.Exists(pluginsFolder))
{
var assemblies = new List <Assembly>();
foreach (var file in Directory.GetFiles(pluginsFolder, "Transformalize.Validate.*.Autofac.dll", SearchOption.TopDirectoryOnly))
{
var info = new FileInfo(file);
var name = info.Name.ToLower().Split('.').FirstOrDefault(f => f != "dll" && f != "transformalize" && f != "validate" && f != "autofac");
loadContext.Debug(() => $"Loading {name} validator(s)");
var assembly = Assembly.LoadFile(new FileInfo(file).FullName);
assemblies.Add(assembly);
}
if (assemblies.Any())
{
builder.RegisterAssemblyModules(assemblies.ToArray());
}
}
// register the short hand
builder.Register((c, p) => _shortHand).Named <ShorthandRoot>(Name).InstancePerLifetimeScope();
builder.Register((c, p) => new ShorthandCustomizer(c.ResolveNamed <ShorthandRoot>(Name), new[] { "fields", "calculated-fields" }, "v", "validators", "method")).Named <IDependency>(Name).InstancePerLifetimeScope();
}
public string Render()
{
var p = new Dictionary <string, string>();
var l = new Cfg.Net.Loggers.MemoryLogger();
// get template
_context.Debug(() => $"Reading {_template.File}");
var templateContent = _templateReader.Read(_template.File, p, l);
if (l.Errors().Any())
{
foreach (var error in l.Errors())
{
_context.Error(error);
}
return(string.Empty);
}
// get parameters (other than process)
var parameters = new ExpandoObject();
foreach (var parameter in _template.Parameters)
{
((IDictionary <string, object>)parameters).Add(parameter.Name, parameter.Value);
}
if (p.Any())
{
foreach (var parameter in p)
{
((IDictionary <string, object>)parameters)[parameter.Key] = parameter.Value;
}
}
try {
_context.Debug(() => $"Compiling {_template.Name}.");
return(_service.RunCompile(templateContent, _template.Name, null, new {
_context.Process,
Parameters = parameters
}));
} catch (Exception ex) {
_context.Error($"Error parsing template {_template.Name}.");
_context.Error(ex, ex.Message.Replace("{", "{{").Replace("}", "}}"));
return(string.Empty);
}
}
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity)
{
builder.Register <IEntityController>(ctx => {
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity);
var output = new OutputContext(context, new Incrementer(context));
switch (output.Connection.Provider)
{
case "sqlserver":
context.Debug("Registering sql server controller");
var initializer = process.Mode == "init" ? (IAction) new SqlEntityInitializer(output) : new NullInitializer();
return(new SqlEntityController(output, initializer));
default:
context.Debug("Registering null controller");
return(new NullEntityController());
}
}).Named <IEntityController>(entity.Key);
}
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity)
{
var type = process.Pipeline == "defer" ? entity.Pipeline : process.Pipeline;
builder.Register <IPipeline>((ctx) => {
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity);
IPipeline pipeline;
switch (type)
{
case "parallel.linq":
context.Debug("Registering {0} pipeline.", type);
pipeline = new ParallelPipeline(new DefaultPipeline(ctx.ResolveNamed <IEntityController>(entity.Key), context));
break;
default:
context.Debug("Registering linq pipeline.", type);
pipeline = new DefaultPipeline(ctx.ResolveNamed <IEntityController>(entity.Key), context);
break;
}
var provider = process.Connections.First(c => c.Name == "output").Provider;
// extract
pipeline.Register(ctx.ResolveNamed <IRead>(entity.Key));
// transform
pipeline.Register(new DefaultTransform(context, context.GetAllEntityFields()));
pipeline.Register(new TflHashCodeTransform(context));
pipeline.Register(TransformFactory.GetTransforms(ctx, process, entity, entity.GetAllFields().Where(f => f.Transforms.Any())));
pipeline.Register(new StringTruncateTransfom(context));
if (provider == "sqlserver")
{
pipeline.Register(new MinDateTransform(context, new DateTime(1753, 1, 1)));
}
//load
pipeline.Register(ctx.ResolveNamed <IWrite>(entity.Key));
pipeline.Register(ctx.ResolveNamed <IUpdate>(entity.Key));
return(pipeline);
}).Named <IPipeline>(entity.Key);
}
protected override void Load(ContainerBuilder builder)
{
if (!builder.Properties.ContainsKey("Process"))
{
return;
}
var process = (Process)builder.Properties["Process"];
// template engines for template actions creating output with an action
foreach (var t in process.Templates.Where(t => t.Enabled && t.Engine == Razor))
{
var template = t;
builder.Register <ITemplateEngine>(ctx => {
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process);
context.Debug(() => $"Registering {template.Engine} Engine for {t.Key}");
return(new RazorTemplateEngine(context, template, ctx.Resolve <IReader>()));
}).Named <ITemplateEngine>(t.Key);
}
// connections, no schema for razor
foreach (var connection in process.Connections.Where(c => c.Provider == Razor))
{
builder.Register <ISchemaReader>(ctx => new NullSchemaReader()).Named <ISchemaReader>(connection.Key);
}
// Entity input
foreach (var entity in process.Entities.Where(e => process.Connections.First(c => c.Name == e.Input).Provider == Razor))
{
// no input version detector
builder.RegisterType <NullInputProvider>().Named <IInputProvider>(entity.Key);
// no input reader
builder.Register <IRead>(ctx => {
var input = ctx.ResolveNamed <InputContext>(entity.Key);
return(new NullReader(input, false));
}).Named <IRead>(entity.Key);
}
if (process.GetOutputConnection().Provider == Razor)
{
// PROCESS OUTPUT CONTROLLER
builder.Register <IOutputController>(ctx => new NullOutputController()).As <IOutputController>();
foreach (var entity in process.Entities)
{
builder.Register <IOutputController>(ctx => new NullOutputController()).Named <IOutputController>(entity.Key);
// ENTITY WRITER
builder.Register <IWrite>(ctx => new RazorWriter(ctx.ResolveNamed <OutputContext>(entity.Key), new DefaultReader(new FileReader(), new WebReader()))).Named <IWrite>(entity.Key);
}
}
}
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) {
var type = process.Pipeline == "defer" ? entity.Pipeline : process.Pipeline;
builder.Register<IPipeline>((ctx) => {
var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity);
IPipeline pipeline;
switch (type) {
case "parallel.linq":
context.Debug("Registering {0} pipeline.", type);
pipeline = new ParallelPipeline(new DefaultPipeline(ctx.ResolveNamed<IEntityController>(entity.Key), context));
break;
default:
context.Debug("Registering linq pipeline.", type);
pipeline = new DefaultPipeline(ctx.ResolveNamed<IEntityController>(entity.Key), context);
break;
}
var provider = process.Connections.First(c => c.Name == "output").Provider;
// extract
pipeline.Register(ctx.ResolveNamed<IRead>(entity.Key));
// transform
pipeline.Register(new DefaultTransform(context, context.GetAllEntityFields()));
pipeline.Register(new TflHashCodeTransform(context));
pipeline.Register(TransformFactory.GetTransforms(ctx,process, entity, entity.GetAllFields().Where(f=>f.Transforms.Any())));
pipeline.Register(new StringTruncateTransfom(context));
if (provider == "sqlserver") {
pipeline.Register(new MinDateTransform(context, new DateTime(1753, 1, 1)));
}
//load
pipeline.Register(ctx.ResolveNamed<IWrite>(entity.Key));
pipeline.Register(ctx.ResolveNamed<IUpdate>(entity.Key));
return pipeline;
}).Named<IPipeline>(entity.Key);
}
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) {
//master updater
builder.Register<IUpdate>((ctx) => {
var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity);
var incrementer = new Incrementer(context);
var output = new OutputContext(context, incrementer);
switch (output.Connection.Provider) {
case "sqlserver":
context.Debug("Registering {0} master updater", output.Connection.Provider);
return new SqlMasterUpdater(output);
default:
context.Warn("Registering null updater");
return new NullMasterUpdater();
}
}).Named<IUpdate>(entity.Key);
}
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity)
{
builder.Register <IWrite>((ctx) => {
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity);
var incrementer = new Incrementer(context);
var output = new OutputContext(context, incrementer);
switch (output.Connection.Provider)
{
case "sqlserver":
context.Debug("Registering Sql Server writer");
return(new SqlEntityBulkInserter(output));
default:
context.Warn("Registering null writer", output.Connection.Provider);
return(new NullWriter(context));
}
}).Named <IWrite>(entity.Key);
}
protected override void Load(ContainerBuilder builder)
{
if (_process == null)
{
return;
}
foreach (var t in _process.Templates.Where(t => t.Enabled))
{
var template = t;
builder.Register <ITemplateEngine>(ctx => {
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), _process);
context.Debug(() => string.Format("Registering {0} Engine for {1}", template.Engine, t.Key));
switch (template.Engine)
{
case "razor":
return(new OrchardRazorTemplateEngine(context, _templateProcessor, template, ctx.Resolve <IReader>()));
default:
return(new NullTemplateEngine());
}
}).Named <ITemplateEngine>(t.Key);
}
}
public static ILifetimeScope Create(Process process, IPipelineLogger logger, string placeHolderStyle)
{
var loadContext = new PipelineContext(logger, process);
if (process.OutputIsConsole())
{
logger.SuppressConsole();
}
var builder = new ContainerBuilder();
builder.Properties["Process"] = process;
builder.Register(ctx => placeHolderStyle).Named <string>("placeHolderStyle");
builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance();
/* this stuff is loaded (again) because tfl actions can create processes, which will need short-hand to expand configuration in advance */
builder.RegisterCallback(new TransformModule(process, logger).Configure);
builder.RegisterCallback(new ValidateModule(process, logger).Configure);
builder.RegisterCallback(new RootModule().Configure);
builder.RegisterCallback(new ContextModule(process).Configure);
// provider loading section
var providers = new HashSet <string>(process.Connections.Select(c => c.Provider).Distinct(), StringComparer.OrdinalIgnoreCase);
builder.RegisterCallback(new InternalModule(process).Configure);
builder.RegisterCallback(new FileModule(process).Configure);
if (providers.Contains("console"))
{
builder.RegisterCallback(new ConsoleModule(process).Configure);
}
if (providers.Contains("kml"))
{
builder.RegisterCallback(new KmlModule(process).Configure);
}
if (providers.Contains("filesystem"))
{
builder.RegisterCallback(new FileSystemModule(process).Configure);
}
var pluginsFolder = Path.Combine(AssemblyDirectory, "plugins");
if (Directory.Exists(pluginsFolder))
{
var assemblies = new List <Assembly>();
var files = Directory.GetFiles(pluginsFolder, "Transformalize.Provider.*.Autofac.dll", SearchOption.TopDirectoryOnly);
foreach (var file in files)
{
var info = new FileInfo(file);
var name = info.Name.ToLower().Split('.').FirstOrDefault(f => f != "dll" && f != "transformalize" && f != "provider" && f != "autofac");
// temporary hack
if (name.StartsWith("amazonkinesis"))
{
name = name.Replace("amazonkinesis", string.Empty);
}
switch (name)
{
case "filehelpers" when(providers.Contains("file") || providers.Contains("folder")):
loadContext.Debug(() => "Loading filehelpers provider");
assemblies.Add(Assembly.LoadFile(new FileInfo(file).FullName));
break;
case "ado":
loadContext.Debug(() => "Loading ADO provider");
assemblies.Add(Assembly.LoadFile(new FileInfo(file).FullName));
break;
default:
if (providers.Contains(name))
{
loadContext.Debug(() => $"Loading {name} provider.");
var assembly = Assembly.LoadFile(new FileInfo(file).FullName);
assemblies.Add(assembly);
}
else
{
loadContext.Debug(() => $"Loading {name} isn't necessary for this arrangement.");
}
break;
}
}
if (assemblies.Any())
{
builder.RegisterAssemblyModules(assemblies.ToArray());
}
}
// etc
builder.RegisterCallback(new EntityPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessControlModule(process).Configure);
return(builder.Build().BeginLifetimeScope());
}
public string Write(EntityStatus status)
{
/* if update occurs, outside of first run, you must update the master's
* batch id to the entity's batch id (which will be higher).
* Updating this indicates a change and we have to update process-level calculated columns
* on the master record.
*
* It also keeps the master's batch id incrementing to indicate
* updates for subsequent processes using this process' output as an input with TflBatchId
* as a version field.
*/
var masterEntity = _c.Process.Entities.First(e => e.IsMaster);
var masterTable = _cf.Enclose(masterEntity.OutputTableName(_c.Process.Name));
var masterAlias = masterEntity.GetExcelName();
var entityAlias = _c.Entity.GetExcelName();
var builder = new StringBuilder();
builder.AppendLine($"UPDATE {masterAlias}");
builder.Append("SET ");
var setPrefix = masterAlias + ".";
foreach (var field in _c.Entity.Fields.Where(f => f.KeyType.HasFlag(KeyType.Foreign)))
{
builder.AppendLine($"{setPrefix}{_cf.Enclose(field.FieldName())} = {entityAlias}.{_cf.Enclose(field.FieldName())},");
}
builder.AppendLine($"{setPrefix}{_cf.Enclose(masterEntity.TflBatchId().FieldName())} = @TflBatchId");
builder.AppendFormat(" FROM {0} {1}", masterTable, masterAlias);
var relationships = _c.Entity.RelationshipToMaster.Reverse().ToArray();
for (var r = 0; r < relationships.Length; r++)
{
var relationship = relationships[r];
var right = _cf.Enclose(relationship.Summary.RightEntity.OutputTableName(_c.Process.Name));
var rightEntityAlias = relationship.Summary.RightEntity.GetExcelName();
builder.AppendFormat(" INNER JOIN {0} {1} ON ( ", right, rightEntityAlias);
var leftEntityAlias = relationship.Summary.LeftEntity.GetExcelName();
for (var i = 0; i < relationship.Summary.LeftFields.Count(); i++)
{
var leftAlias = relationship.Summary.LeftFields[i].FieldName();
var rightAlias = relationship.Summary.RightFields[i].FieldName();
var conjunction = i > 0 ? " AND " : string.Empty;
builder.AppendFormat(
"{0}{1}.{2} = {3}.{4}",
conjunction,
leftEntityAlias,
_cf.Enclose(leftAlias),
rightEntityAlias,
_cf.Enclose(rightAlias)
);
}
builder.AppendLine(")");
}
builder.Append("WHERE ");
if (status.Modified)
{
builder.AppendLine($"{entityAlias}.{_cf.Enclose(_c.Entity.TflBatchId().FieldName())} = @TflBatchId ");
if (status.MasterUpserted && status.HasForeignKeys)
{
builder.AppendLine($"OR {masterAlias}.{_cf.Enclose(masterEntity.TflBatchId().FieldName())} >= @MasterTflBatchId");
}
}
else if (status.MasterUpserted && status.HasForeignKeys)
{
builder.AppendLine($"{masterAlias}.{_cf.Enclose(masterEntity.TflBatchId().FieldName())} >= @MasterTflBatchId");
}
var sql = builder.ToString();
_c.Debug(() => sql);
return(sql);
}
protected override void Load(ContainerBuilder builder)
{
var loadContext = new PipelineContext(_logger, _process);
builder.Properties["ShortHand"] = _shortHand;
builder.Properties["Methods"] = _methods;
builder.Properties["Process"] = _process;
// new method so transform author can define shorthand signature(s)
RegisterTransform(builder, c => new AbsTransform(c), new AbsTransform().GetSignatures());
RegisterTransform(builder, c => new SliceTransform(c), new SliceTransform().GetSignatures());
RegisterTransform(builder, c => new FormatTransform(c), new FormatTransform().GetSignatures());
RegisterTransform(builder, c => new AddTransform(c), new AddTransform().GetSignatures());
RegisterTransform(builder, c => new EqualsTransform(c), new EqualsTransform().GetSignatures());
RegisterTransform(builder, c => new CeilingTransform(c), new CeilingTransform().GetSignatures());
RegisterTransform(builder, c => new CoalesceTransform(c), new CoalesceTransform().GetSignatures());
RegisterTransform(builder, c => new ConcatTransform(c), new ConcatTransform().GetSignatures());
RegisterTransform(builder, c => new ConnectionTransform(c), new ConnectionTransform().GetSignatures());
RegisterTransform(builder, c => new ConvertTransform(c), new ConvertTransform().GetSignatures());
RegisterTransform(builder, c => new CopyTransform(c), new CopyTransform().GetSignatures());
RegisterTransform(builder, c => new DateDiffTransform(c), new DateDiffTransform().GetSignatures());
RegisterTransform(builder, c => new DatePartTransform(c), new DatePartTransform().GetSignatures());
RegisterTransform(builder, c => new DecompressTransform(c), new DecompressTransform().GetSignatures());
RegisterTransform(builder, c => new CompressTransform(c), new CompressTransform().GetSignatures());
RegisterTransform(builder, c => new FileExtTransform(c), new FileExtTransform().GetSignatures());
RegisterTransform(builder, c => new FileNameTransform(c), new FileNameTransform().GetSignatures());
RegisterTransform(builder, c => new FilePathTransform(c), new FilePathTransform().GetSignatures());
RegisterTransform(builder, c => new SplitTransform(c), new SplitTransform().GetSignatures());
RegisterTransform(builder, c => new LastDayTransform(c), new LastDayTransform().GetSignatures());
RegisterTransform(builder, c => new LastTransform(c), new LastTransform().GetSignatures());
RegisterTransform(builder, c => new FirstTransform(c), new FirstTransform().GetSignatures());
RegisterTransform(builder, c => new FloorTransform(c), new FloorTransform().GetSignatures());
RegisterTransform(builder, c => new FormatXmlTransform(c), new FormatXmlTransform().GetSignatures());
RegisterTransform(builder, c => new FormatPhoneTransform(c), new FormatPhoneTransform().GetSignatures());
RegisterTransform(builder, c => new HashcodeTransform(c), new HashcodeTransform().GetSignatures());
RegisterTransform(builder, c => new DecodeTransform(c), new DecodeTransform().GetSignatures());
RegisterTransform(builder, c => new HtmlEncodeTransform(c), new HtmlEncodeTransform().GetSignatures());
RegisterTransform(builder, c => new InsertTransform(c), new InsertTransform().GetSignatures());
RegisterTransform(builder, c => new InvertTransform(c), new InvertTransform().GetSignatures());
RegisterTransform(builder, c => new JoinTransform(c), new JoinTransform().GetSignatures());
RegisterTransform(builder, c => new LeftTransform(c), new LeftTransform().GetSignatures());
RegisterTransform(builder, c => new RightTransform(c), new RightTransform().GetSignatures());
RegisterTransform(builder, c => new ToLowerTransform(c), new ToLowerTransform().GetSignatures());
RegisterTransform(builder, c => new MapTransform(c), new MapTransform().GetSignatures());
RegisterTransform(builder, c => new RegexMatchTransform(c), new RegexMatchTransform().GetSignatures());
RegisterTransform(builder, c => new RegexMatchingTransform(c), new RegexMatchingTransform().GetSignatures());
RegisterTransform(builder, c => new MultiplyTransform(c), new MultiplyTransform().GetSignatures());
RegisterTransform(builder, c => new NextTransform(c), new NextTransform().GetSignatures());
RegisterTransform(builder, c => new UtcNowTransform(c), new UtcNowTransform().GetSignatures());
RegisterTransform(builder, c => new RowNumberTransform(c), new RowNumberTransform().GetSignatures());
RegisterTransform(builder, c => new TimeZoneTransform(c), new TimeZoneTransform().GetSignatures());
RegisterTransform(builder, c => new TimeZoneOffsetTransform(c), new TimeZoneOffsetTransform().GetSignatures());
RegisterTransform(builder, c => new OppositeTransform(c), new OppositeTransform().GetSignatures());
RegisterTransform(builder, c => new DateAddTransform(c), new DateAddTransform().GetSignatures());
RegisterTransform(builder, c => new TimeAheadTransform(c), new TimeAheadTransform().GetSignatures());
RegisterTransform(builder, c => new TimeAgoTransform(c), new TimeAgoTransform().GetSignatures());
var pluginsFolder = Path.Combine(AssemblyDirectory, "plugins");
if (Directory.Exists(pluginsFolder))
{
var assemblies = new List <Assembly>();
foreach (var file in Directory.GetFiles(pluginsFolder, "Transformalize.Transform.*.Autofac.dll", SearchOption.TopDirectoryOnly))
{
var info = new FileInfo(file);
var name = info.Name.ToLower().Split('.').FirstOrDefault(f => f != "dll" && f != "transformalize" && f != "transform" && f != "autofac");
loadContext.Debug(() => $"Loading {name} transform");
var assembly = Assembly.LoadFile(new FileInfo(file).FullName);
assemblies.Add(assembly);
}
if (assemblies.Any())
{
builder.RegisterAssemblyModules(assemblies.ToArray());
}
}
// register the short hand
builder.Register((c, p) => _shortHand).Named <ShorthandRoot>(Name).InstancePerLifetimeScope();
// old method
builder.Register((c, p) => new PadLeftTransform(p.Positional <IContext>(0))).Named <ITransform>("padleft");
builder.Register((c, p) => new PadRightTransform(p.Positional <IContext>(0))).Named <ITransform>("padright");
builder.Register((c, p) => new RazorTransform(p.Positional <IContext>(0))).Named <ITransform>("razor");
builder.Register((c, p) => new RegexReplaceTransform(p.Positional <IContext>(0))).Named <ITransform>("regexreplace");
builder.Register((c, p) => new RemoveTransform(p.Positional <IContext>(0))).Named <ITransform>("remove");
builder.Register((c, p) => new ReplaceTransform(p.Positional <IContext>(0))).Named <ITransform>("replace");
builder.Register((c, p) => new RoundTransform(p.Positional <IContext>(0))).Named <ITransform>("round");
builder.Register((c, p) => new RoundToTransform(p.Positional <IContext>(0), RoundTo.Nearest)).Named <ITransform>("roundto");
builder.Register((c, p) => new RoundToTransform(p.Positional <IContext>(0), RoundTo.Up)).Named <ITransform>("roundupto");
builder.Register((c, p) => new RoundToTransform(p.Positional <IContext>(0), RoundTo.Down)).Named <ITransform>("rounddownto");
builder.Register((c, p) => new SplitLengthTransform(p.Positional <IContext>(0))).Named <ITransform>("splitlength");
builder.Register((c, p) => new SubStringTransform(p.Positional <IContext>(0))).Named <ITransform>("substring");
builder.Register((c, p) => new TagTransform(p.Positional <IContext>(0))).Named <ITransform>("tag");
builder.Register((c, p) => new ToStringTransform(p.Positional <IContext>(0))).Named <ITransform>("tostring");
builder.Register((c, p) => new ToTimeTransform(p.Positional <IContext>(0))).Named <ITransform>("totime");
builder.Register((c, p) => new ToYesNoTransform(p.Positional <IContext>(0))).Named <ITransform>("toyesno");
builder.Register((c, p) => new TrimTransform(p.Positional <IContext>(0))).Named <ITransform>("trim");
builder.Register((c, p) => new TrimEndTransform(p.Positional <IContext>(0))).Named <ITransform>("trimend");
builder.Register((c, p) => new TrimStartTransform(p.Positional <IContext>(0))).Named <ITransform>("trimstart");
builder.Register((c, p) => new VelocityTransform(p.Positional <IContext>(0), c.Resolve <IReader>())).Named <ITransform>("velocity");
builder.Register((c, p) => new ToUpperTransform(p.Positional <IContext>(0))).Named <ITransform>("upper");
builder.Register((c, p) => new ToUpperTransform(p.Positional <IContext>(0))).Named <ITransform>("toupper");
builder.Register((c, p) => new XPathTransform(p.Positional <IContext>(0))).Named <ITransform>("xpath");
builder.Register((c, p) => new IIfTransform(p.Positional <IContext>(0))).Named <ITransform>("iif");
builder.Register((c, p) => new GeohashEncodeTransform(p.Positional <IContext>(0))).Named <ITransform>("geohashencode");
builder.Register((c, p) => new GeohashNeighborTransform(p.Positional <IContext>(0))).Named <ITransform>("geohashneighbor");
builder.Register((c, p) => new CommonPrefixTransform(p.Positional <IContext>(0))).Named <ITransform>("commonprefix");
builder.Register((c, p) => new CommonPrefixesTransform(p.Positional <IContext>(0))).Named <ITransform>("commonprefixes");
builder.Register((c, p) => new DistanceTransform(p.Positional <IContext>(0))).Named <ITransform>("distance");
builder.Register((c, p) => new FilterTransform(p.Positional <IContext>(0), FilterType.Include)).Named <ITransform>("include");
builder.Register((c, p) => new FilterTransform(p.Positional <IContext>(0), FilterType.Exclude)).Named <ITransform>("exclude");
builder.Register((c, p) => new FromSplitTransform(p.Positional <IContext>(0))).Named <ITransform>("fromsplit");
builder.Register((c, p) => new FromRegexTransform(p.Positional <IContext>(0))).Named <ITransform>("fromregex");
builder.Register((c, p) => new FromLengthsTranform(p.Positional <IContext>(0))).Named <ITransform>("fromlengths");
// return true or false, validators
builder.Register((c, p) => new AnyTransform(p.Positional <IContext>(0))).Named <ITransform>("any");
builder.Register((c, p) => new StartsWithTransform(p.Positional <IContext>(0))).Named <ITransform>("startswith");
builder.Register((c, p) => new EndsWithTransform(p.Positional <IContext>(0))).Named <ITransform>("endswith");
builder.Register((c, p) => new InTransform(p.Positional <IContext>(0))).Named <ITransform>("in");
builder.Register((c, p) => new ContainsTransform(p.Positional <IContext>(0))).Named <ITransform>("contains");
builder.Register((c, p) => new IsTransform(p.Positional <IContext>(0))).Named <ITransform>("is");
builder.Register((c, p) => new EqualsTransform(p.Positional <IContext>(0))).Named <ITransform>("equal");
builder.Register((c, p) => new EqualsTransform(p.Positional <IContext>(0))).Named <ITransform>("equals");
builder.Register((c, p) => new IsEmptyTransform(p.Positional <IContext>(0))).Named <ITransform>("isempty");
builder.Register((c, p) => new IsDefaultTransform(p.Positional <IContext>(0))).Named <ITransform>("isdefault");
builder.Register((c, p) => new IsNumericTransform(p.Positional <IContext>(0))).Named <ITransform>("isnumeric");
builder.Register((c, p) => new RegexIsMatchTransform(p.Positional <IContext>(0))).Named <ITransform>("ismatch");
builder.Register((c, p) => new GeocodeTransform(p.Positional <IContext>(0))).Named <ITransform>("fromaddress");
builder.Register((c, p) => new DateMathTransform(p.Positional <IContext>(0))).Named <ITransform>("datemath");
builder.Register((c, p) => new IsDaylightSavingsTransform(p.Positional <IContext>(0))).Named <ITransform>("isdaylightsavings");
builder.Register((c, p) => new SlugifyTransform(p.Positional <IContext>(0))).Named <ITransform>("slugify");
// wip
builder.Register((c, p) => new WebTransform(p.Positional <IContext>(0))).Named <ITransform>("web");
builder.Register((c, p) => new UrlEncodeTransform(p.Positional <IContext>(0))).Named <ITransform>("urlencode");
builder.Register((c, p) => new FromJsonTransform(p.Positional <IContext>(0), o => JsonConvert.SerializeObject(o, Formatting.None))).Named <ITransform>("fromjson");
builder.Register((c, p) => new DistinctTransform(p.Positional <IContext>(0))).Named <ITransform>("distinct");
builder.Register((c, p) => new RegexMatchCountTransform(p.Positional <IContext>(0))).Named <ITransform>("matchcount");
builder.Register((c, p) => new AppendTransform(p.Positional <IContext>(0))).Named <ITransform>("append");
builder.Register((c, p) => new PrependTransform(p.Positional <IContext>(0))).Named <ITransform>("prepend");
builder.Register((c, p) => new AppendTransform(p.Positional <IContext>(0))).Named <ITransform>("prefix");
builder.Register((c, p) => new PrependTransform(p.Positional <IContext>(0))).Named <ITransform>("suffix");
builder.Register((c, p) => {
var context = p.Positional <IContext>(0);
return(context.Operation.Mode == "all" || context.Field.Engine != "auto" ?
new Transforms.Xml.FromXmlTransform(context, c.ResolveNamed <IRowFactory>(context.Entity.Key, new NamedParameter("capacity", context.GetAllEntityFields().Count()))) :
new Transforms.FromXmlTransform(context) as ITransform);
}).Named <ITransform>("fromxml");
builder.Register((c, p) => new JavascriptTransform(new ChakraCoreJsEngineFactory(), p.Positional <IContext>(0), c.Resolve <IReader>())).Named <ITransform>("js");
builder.Register((c, p) => new JavascriptTransform(new ChakraCoreJsEngineFactory(), p.Positional <IContext>(0), c.Resolve <IReader>())).Named <ITransform>("javascript");
builder.Register((c, p) => new JavascriptTransform(new ChakraCoreJsEngineFactory(), p.Positional <IContext>(0), c.Resolve <IReader>())).Named <ITransform>("chakra");
}
protected BaseTransform(PipelineContext context)
{
context.Activity = PipelineActivity.Transform;
Context = context;
Context.Debug("Registered");
}
public ILifetimeScope CreateScope(Process process, IPipelineLogger logger)
{
var builder = new ContainerBuilder();
#if PLUGINS
builder.Properties["Process"] = process;
#endif
builder.Register(ctx => process).As <Process>();
builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance();
// register short-hand for t attribute, allowing for additional transforms
var transformModule = new TransformModule(process, _methods, _shortHand, logger);
foreach (var t in _transforms)
{
transformModule.AddTransform(t);
}
builder.RegisterModule(transformModule);
// register short-hand for v attribute, allowing for additional validators
var validateModule = new ValidateModule(process, _methods, _shortHand, logger);
foreach (var v in _validators)
{
validateModule.AddValidator(v);
}
builder.RegisterModule(validateModule);
builder.RegisterModule(new InternalModule(process));
#if PLUGINS
// just in case other modules need to see these
builder.Properties["ShortHand"] = _shortHand;
builder.Properties["Methods"] = _methods;
#endif
// allowing for additional modules
foreach (var module in _modules)
{
builder.RegisterModule(module);
}
// Process Context
builder.Register <IContext>((ctx, p) => new PipelineContext(logger, process)).As <IContext>();
// Process Output Context
builder.Register(ctx => {
var context = ctx.Resolve <IContext>();
return(new OutputContext(context));
}).As <OutputContext>();
// Connection and Process Level Output Context
foreach (var connection in process.Connections)
{
builder.Register(ctx => new ConnectionContext(ctx.Resolve <IContext>(), connection)).Named <IConnectionContext>(connection.Key);
if (connection.Name != "output")
{
continue;
}
// register output for connection
builder.Register(ctx => {
var context = ctx.ResolveNamed <IConnectionContext>(connection.Key);
return(new OutputContext(context));
}).Named <OutputContext>(connection.Key);
}
// Entity Context and RowFactory
foreach (var entity in process.Entities)
{
builder.Register <IContext>((ctx, p) => new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity)).Named <IContext>(entity.Key);
builder.Register(ctx => {
var context = ctx.ResolveNamed <IContext>(entity.Key);
return(new InputContext(context));
}).Named <InputContext>(entity.Key);
builder.Register <IRowFactory>((ctx, p) => new RowFactory(p.Named <int>("capacity"), entity.IsMaster, false)).Named <IRowFactory>(entity.Key);
builder.Register(ctx => {
var context = ctx.ResolveNamed <IContext>(entity.Key);
return(new OutputContext(context));
}).Named <OutputContext>(entity.Key);
var connection = process.Connections.First(c => c.Name == entity.Connection);
builder.Register(ctx => new ConnectionContext(ctx.Resolve <IContext>(), connection)).Named <IConnectionContext>(entity.Key);
}
// entity pipelines
foreach (var entity in process.Entities)
{
builder.Register(ctx => {
var type = process.Pipeline == "defer" ? entity.Pipeline : process.Pipeline;
var context = ctx.ResolveNamed <IContext>(entity.Key);
context.Debug(() => $"Registering {type} for entity {entity.Alias}.");
var outputController = ctx.IsRegisteredWithName <IOutputController>(entity.Key) ? ctx.ResolveNamed <IOutputController>(entity.Key) : new NullOutputController();
var pipeline = new DefaultPipeline(outputController, context);
// TODO: rely on IInputProvider's Read method instead (after every provider has one)
pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IRead)) ? ctx.ResolveNamed <IRead>(entity.Key) : null);
pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IInputProvider)) ? ctx.ResolveNamed <IInputProvider>(entity.Key) : null);
// transforms
if (!process.ReadOnly)
{
pipeline.Register(new SetSystemFields(new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity)));
}
pipeline.Register(new IncrementTransform(context));
pipeline.Register(new DefaultTransform(context, context.GetAllEntityFields().Where(f => !f.System)));
pipeline.Register(TransformFactory.GetTransforms(ctx, context, entity.GetAllFields().Where(f => f.Transforms.Any())));
pipeline.Register(ValidateFactory.GetValidators(ctx, context, entity.GetAllFields().Where(f => f.Validators.Any())));
if (!process.ReadOnly)
{
pipeline.Register(new StringTruncateTransfom(new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity)));
}
pipeline.Register(new LogTransform(context));
// writer, TODO: rely on IOutputProvider instead
pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IWrite)) ? ctx.ResolveNamed <IWrite>(entity.Key) : null);
pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IOutputProvider)) ? ctx.ResolveNamed <IOutputProvider>(entity.Key) : null);
// updater
pipeline.Register(process.ReadOnly || !ctx.IsRegisteredWithName(entity.Key, typeof(IUpdate)) ? new NullUpdater() : ctx.ResolveNamed <IUpdate>(entity.Key));
return(pipeline);
}).Named <IPipeline>(entity.Key);
}
// process pipeline
builder.Register(ctx => {
var calc = process.ToCalculatedFieldsProcess();
var entity = calc.Entities.First();
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity);
var outputContext = new OutputContext(context);
context.Debug(() => $"Registering {process.Pipeline} pipeline.");
var outputController = ctx.IsRegistered <IOutputController>() ? ctx.Resolve <IOutputController>() : new NullOutputController();
var pipeline = new DefaultPipeline(outputController, context);
// no updater necessary
pipeline.Register(new NullUpdater(context, false));
if (!process.CalculatedFields.Any())
{
pipeline.Register(new NullReader(context, false));
pipeline.Register(new NullWriter(context, false));
return(pipeline);
}
// register transforms
pipeline.Register(new IncrementTransform(context));
pipeline.Register(new LogTransform(context));
pipeline.Register(new DefaultTransform(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity), entity.CalculatedFields));
pipeline.Register(TransformFactory.GetTransforms(ctx, context, entity.CalculatedFields));
pipeline.Register(ValidateFactory.GetValidators(ctx, context, entity.GetAllFields().Where(f => f.Validators.Any())));
pipeline.Register(new StringTruncateTransfom(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity)));
// register input and output
pipeline.Register(ctx.IsRegistered <IRead>() ? ctx.Resolve <IRead>() : new NullReader(context));
pipeline.Register(ctx.IsRegistered <IWrite>() ? ctx.Resolve <IWrite>() : new NullWriter(context));
if (outputContext.Connection.Provider == "sqlserver")
{
pipeline.Register(new MinDateTransform(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity), new DateTime(1753, 1, 1)));
}
return(pipeline);
}).As <IPipeline>();
// process controller
builder.Register <IProcessController>(ctx => {
var pipelines = new List <IPipeline>();
// entity-level pipelines
foreach (var entity in process.Entities)
{
var pipeline = ctx.ResolveNamed <IPipeline>(entity.Key);
pipelines.Add(pipeline);
if (entity.Delete && process.Mode != "init")
{
pipeline.Register(ctx.ResolveNamed <IEntityDeleteHandler>(entity.Key));
}
}
// process-level pipeline for process level calculated fields
if (ctx.IsRegistered <IPipeline>())
{
pipelines.Add(ctx.Resolve <IPipeline>());
}
var context = ctx.Resolve <IContext>();
var controller = new ProcessController(pipelines, context);
// output initialization
if (process.Mode == "init" && ctx.IsRegistered <IInitializer>())
{
controller.PreActions.Add(ctx.Resolve <IInitializer>());
}
// flatten(ing) is first post-action
var isAdo = Constants.AdoProviderSet().Contains(process.Output().Provider);
if (process.Flatten && isAdo)
{
if (ctx.IsRegisteredWithName <IAction>(process.Output().Key))
{
controller.PostActions.Add(ctx.ResolveNamed <IAction>(process.Output().Key));
}
else
{
context.Error($"Could not find ADO Flatten Action for provider {process.Output().Provider}.");
}
}
// actions
foreach (var action in process.Actions.Where(a => a.GetModes().Any(m => m == process.Mode || m == "*")))
{
if (action.Before)
{
controller.PreActions.Add(ctx.ResolveNamed <IAction>(action.Key));
}
if (action.After)
{
controller.PostActions.Add(ctx.ResolveNamed <IAction>(action.Key));
}
}
return(controller);
}).As <IProcessController>();
var build = builder.Build();
return(build.BeginLifetimeScope());
}
protected override void RegisterProcess(ContainerBuilder builder, Process original)
{
// I need to create a new process with an entity with the appropriate fields
// clone process, remove entities, and create entity needed for calcuted fields
var calc = original.Clone() as Process;
calc.Entities.Clear();
calc.CalculatedFields.Clear();
calc.Relationships.Clear();
var entity = original.GetDefaultOf <Entity>(e => {
e.Name = "sys.Calc";
e.Connection = "output";
e.IsMaster = false;
e.Fields = new List <Field> {
original.GetDefaultOf <Field>(f => {
f.Name = Constants.TflKey;
f.Type = "int";
f.PrimaryKey = true;
})
};
});
// Add fields that calculated fields depend on
entity.Fields.AddRange(original.CalculatedFields
.SelectMany(f => f.Transforms)
.SelectMany(t => t.Parameters)
.Where(p => !p.HasValue() && p.IsField(original))
.Select(p => p.AsField(original).Clone() as Field)
);
entity.CalculatedFields.AddRange(original.CalculatedFields.Select(cf => cf.Clone() as Field));
calc.Entities.Add(entity);
calc.ModifyKeys();
calc.ModifyIndexes();
// I need a process keyed pipeline
builder.Register((ctx) => {
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity);
IPipeline pipeline;
switch (original.Pipeline)
{
case "parallel.linq":
context.Debug("Registering {0} pipeline.", original.Pipeline);
pipeline = new ParallelPipeline(new DefaultPipeline(new NullEntityController(), context));
break;
default:
context.Debug("Registering linq pipeline.", original.Pipeline);
pipeline = new DefaultPipeline(new NullEntityController(), context);
break;
}
// register transforms
pipeline.Register(new DefaultTransform(context, entity.CalculatedFields));
pipeline.Register(TransformFactory.GetTransforms(ctx, calc, entity, entity.CalculatedFields));
pipeline.Register(new StringTruncateTransfom(context));
// register input and output
var outputContext = new OutputContext(context, new Incrementer(context));
switch (outputContext.Connection.Provider)
{
case "sqlserver":
pipeline.Register(new SqlStarParametersReader(outputContext, original));
pipeline.Register(new SqlCalculatedFieldUpdater(outputContext, original));
pipeline.Register(new MinDateTransform(context, new DateTime(1753, 1, 1)));
break;
default:
pipeline.Register(new NullReader(context));
pipeline.Register(new NullWriter(context));
break;
}
// no updater necessary
pipeline.Register(new NullUpdater(context, false));
return(pipeline);
}).Named <IPipeline>(original.Key);
}
protected override void Load(ContainerBuilder builder)
{
if (_process == null)
{
return;
}
var calc = _process.ToCalculatedFieldsProcess();
var entity = calc.Entities.First();
// I need a process keyed pipeline
builder.Register(ctx => {
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity);
var outputContext = new OutputContext(context, new Incrementer(context));
IPipeline pipeline;
context.Debug(() => $"Registering {_process.Pipeline} pipeline.");
var outputController = ctx.IsRegistered <IOutputController>() ? ctx.Resolve <IOutputController>() : new NullOutputController();
switch (_process.Pipeline)
{
case "parallel.linq":
pipeline = new ParallelPipeline(new DefaultPipeline(outputController, context));
break;
default:
pipeline = new DefaultPipeline(outputController, context);
break;
}
// no updater necessary
pipeline.Register(new NullUpdater(context, false));
if (!_process.CalculatedFields.Any())
{
pipeline.Register(new NullReader(context, false));
pipeline.Register(new NullWriter(context, false));
return(pipeline);
}
// register transforms
pipeline.Register(new DefaultTransform(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity), entity.CalculatedFields));
pipeline.Register(TransformFactory.GetTransforms(ctx, calc, entity, entity.CalculatedFields));
pipeline.Register(new StringTruncateTransfom(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity)));
// register input and output
switch (outputContext.Connection.Provider)
{
case "sqlserver":
pipeline.Register(ctx.Resolve <IRead>());
pipeline.Register(ctx.Resolve <IWrite>());
pipeline.Register(new MinDateTransform(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity), new DateTime(1753, 1, 1)));
break;
case "mysql":
case "postgresql":
case "sqlce":
case "sqlite":
pipeline.Register(ctx.Resolve <IRead>());
pipeline.Register(ctx.Resolve <IWrite>());
break;
default:
pipeline.Register(new NullReader(context));
pipeline.Register(new NullWriter(context));
break;
}
return(pipeline);
}).As <IPipeline>();
}
protected BaseTransform(PipelineContext context) {
context.Activity = PipelineActivity.Transform;
Context = context;
Context.Debug("Registered");
}
public static ILifetimeScope Create(Process process, IPipelineLogger logger, string placeHolderStyle)
{
var loadContext = new PipelineContext(logger, process);
if (process.OutputIsConsole())
{
logger.SuppressConsole();
}
var builder = new ContainerBuilder();
builder.Properties["Process"] = process;
builder.Register(ctx => placeHolderStyle).Named <string>("placeHolderStyle");
builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance();
/* this stuff is loaded (again) because tfl actions can create processes, which will need short-hand to expand configuration in advance */
builder.RegisterCallback(new TransformModule(process, logger).Configure);
builder.RegisterCallback(new ShorthandTransformModule().Configure);
builder.RegisterCallback(new ValidateModule().Configure);
builder.RegisterCallback(new ShorthandValidateModule().Configure);
builder.RegisterCallback(new RootModule().Configure);
builder.RegisterCallback(new ContextModule(process).Configure);
// provider loading section
var providers = new HashSet <string>(process.Connections.Select(c => c.Provider).Distinct(), StringComparer.OrdinalIgnoreCase);
builder.RegisterCallback(new InternalModule(process).Configure);
builder.RegisterCallback(new AdoModule(process).Configure);
if (providers.Contains("console"))
{
builder.RegisterCallback(new ConsoleModule(process).Configure);
}
if (providers.Contains("file"))
{
builder.RegisterCallback(new FileModule(process).Configure);
}
if (providers.Contains("geojson"))
{
builder.RegisterCallback(new GeoJsonModule(process).Configure);
}
if (providers.Contains("kml"))
{
builder.RegisterCallback(new KmlModule(process).Configure);
}
if (providers.Contains("folder"))
{
builder.RegisterCallback(new FolderModule(process).Configure);
}
if (providers.Contains("filesystem"))
{
builder.RegisterCallback(new FileSystemModule(process).Configure);
}
if (providers.Contains("excel"))
{
builder.RegisterCallback(new ExcelModule(process).Configure);
}
if (providers.Contains("web"))
{
builder.RegisterCallback(new WebModule(process).Configure);
}
if (providers.Contains("rethinkdb"))
{
builder.RegisterCallback(new RethinkDBModule(process).Configure);
}
var pluginsFolder = Path.Combine(AssemblyDirectory, "plugins");
if (Directory.Exists(pluginsFolder))
{
var assemblies = new List <Assembly>();
foreach (var file in Directory.GetFiles(pluginsFolder, "Transformalize.Provider.*.Autofac.dll", SearchOption.TopDirectoryOnly))
{
var info = new FileInfo(file);
var name = info.Name.ToLower().Split('.').FirstOrDefault(f => f != "dll" && f != "transformalize" && f != "provider" && f != "autofac");
if (!providers.Contains(name))
{
continue;
}
loadContext.Debug(() => $"Loading {name} provider");
var assembly = Assembly.LoadFile(new FileInfo(file).FullName);
assemblies.Add(assembly);
}
if (assemblies.Any())
{
builder.RegisterAssemblyModules(assemblies.ToArray());
}
}
// template providers
builder.RegisterCallback(new RazorModule(process).Configure);
// etc
builder.RegisterCallback(new MapModule(process).Configure);
builder.RegisterCallback(new TemplateModule(process).Configure);
builder.RegisterCallback(new ActionModule(process).Configure);
builder.RegisterCallback(new EntityPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessControlModule(process).Configure);
return(builder.Build().BeginLifetimeScope());
}
protected override void RegisterProcess(ContainerBuilder builder, Process original) {
// I need to create a new process with an entity with the appropriate fields
// clone process, remove entities, and create entity needed for calcuted fields
var calc = original.Clone() as Process;
calc.Entities.Clear();
calc.CalculatedFields.Clear();
calc.Relationships.Clear();
var entity = original.GetDefaultOf<Entity>(e => {
e.Name = "sys.Calc";
e.Connection = "output";
e.IsMaster = false;
e.Fields = new List<Field> {
original.GetDefaultOf<Field>(f => {
f.Name = Constants.TflKey;
f.Type = "int";
f.PrimaryKey = true;
})
};
});
// Add fields that calculated fields depend on
entity.Fields.AddRange(original.CalculatedFields
.SelectMany(f => f.Transforms)
.SelectMany(t => t.Parameters)
.Where(p => !p.HasValue() && p.IsField(original))
.Select(p => p.AsField(original).Clone() as Field)
);
entity.CalculatedFields.AddRange(original.CalculatedFields.Select(cf => cf.Clone() as Field));
calc.Entities.Add(entity);
calc.ModifyKeys();
calc.ModifyIndexes();
// I need a process keyed pipeline
builder.Register((ctx) => {
var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), calc, entity);
IPipeline pipeline;
switch (original.Pipeline) {
case "parallel.linq":
context.Debug("Registering {0} pipeline.", original.Pipeline);
pipeline = new ParallelPipeline(new DefaultPipeline(new NullEntityController(), context));
break;
default:
context.Debug("Registering linq pipeline.", original.Pipeline);
pipeline = new DefaultPipeline(new NullEntityController(), context);
break;
}
// register transforms
pipeline.Register(new DefaultTransform(context, entity.CalculatedFields));
pipeline.Register(TransformFactory.GetTransforms(ctx, calc, entity, entity.CalculatedFields));
pipeline.Register(new StringTruncateTransfom(context));
// register input and output
var outputContext = new OutputContext(context, new Incrementer(context));
switch (outputContext.Connection.Provider) {
case "sqlserver":
pipeline.Register(new SqlStarParametersReader(outputContext, original));
pipeline.Register(new SqlCalculatedFieldUpdater(outputContext, original));
pipeline.Register(new MinDateTransform(context, new DateTime(1753, 1, 1)));
break;
default:
pipeline.Register(new NullReader(context));
pipeline.Register(new NullWriter(context));
break;
}
// no updater necessary
pipeline.Register(new NullUpdater(context, false));
return pipeline;
}).Named<IPipeline>(original.Key);
}
public ILifetimeScope CreateScope(ILifetimeScope scope, IPipelineLogger logger)
{
var process = scope.Resolve <Process>();
var builder = new ContainerBuilder();
builder.Properties["Process"] = process;
builder.Register((ctx) => process).As <Process>();
builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance();
builder.RegisterModule(new CsScriptModule());
builder.RegisterModule(new BogusModule());
RegisterTransform(builder, c => new HashcodeTransform(c), new[] { new OperationSignature("hashcode") });
// Process Context
builder.Register <IContext>((ctx, p) => new PipelineContext(logger, process)).As <IContext>();
// Process Output Context
builder.Register(ctx => {
var context = ctx.Resolve <IContext>();
return(new OutputContext(context));
}).As <OutputContext>();
// Connection and Process Level Output Context
foreach (var connection in process.Connections)
{
builder.Register(ctx => new ConnectionContext(ctx.Resolve <IContext>(), connection)).Named <IConnectionContext>(connection.Key);
if (connection.Name != "output")
{
continue;
}
// register output for connection
builder.Register(ctx => {
var context = ctx.ResolveNamed <IConnectionContext>(connection.Key);
return(new OutputContext(context));
}).Named <OutputContext>(connection.Key);
}
// Entity Context and RowFactory
foreach (var entity in process.Entities)
{
builder.Register <IContext>((ctx, p) => new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity)).Named <IContext>(entity.Key);
builder.Register(ctx => {
var context = ctx.ResolveNamed <IContext>(entity.Key);
return(new InputContext(context));
}).Named <InputContext>(entity.Key);
builder.Register <IRowFactory>((ctx, p) => new RowFactory(p.Named <int>("capacity"), entity.IsMaster, false)).Named <IRowFactory>(entity.Key);
builder.Register(ctx => {
var context = ctx.ResolveNamed <IContext>(entity.Key);
return(new OutputContext(context));
}).Named <OutputContext>(entity.Key);
var connection = process.Connections.First(c => c.Name == entity.Connection);
builder.Register(ctx => new ConnectionContext(ctx.Resolve <IContext>(), connection)).Named <IConnectionContext>(entity.Key);
}
// Entity input
foreach (var entity in process.Entities.Where(e => process.Connections.First(c => c.Name == e.Connection).Provider == "internal"))
{
builder.RegisterType <NullInputProvider>().Named <IInputProvider>(entity.Key);
// READER
builder.Register <IRead>(ctx => {
var input = ctx.ResolveNamed <InputContext>(entity.Key);
var rowFactory = ctx.ResolveNamed <IRowFactory>(entity.Key, new NamedParameter("capacity", input.RowCapacity));
return(new InternalReader(input, rowFactory));
}).Named <IRead>(entity.Key);
}
// PROCESS OUTPUT CONTROLLER
builder.Register <IOutputController>(ctx => new NullOutputController()).As <IOutputController>();
// Entity Output
foreach (var entity in process.Entities)
{
builder.Register <IOutputController>(ctx => new NullOutputController()).Named <IOutputController>(entity.Key);
builder.Register <IOutputProvider>(ctx => new InternalOutputProvider(ctx.ResolveNamed <OutputContext>(entity.Key), ctx.ResolveNamed <IWrite>(entity.Key))).Named <IOutputProvider>(entity.Key);
// WRITER
builder.Register <IWrite>(ctx => new InternalWriter(ctx.ResolveNamed <OutputContext>(entity.Key))).Named <IWrite>(entity.Key);
}
// entity pipelines
foreach (var entity in process.Entities)
{
builder.Register(ctx => {
var context = ctx.ResolveNamed <IContext>(entity.Key);
var outputController = ctx.IsRegisteredWithName <IOutputController>(entity.Key) ? ctx.ResolveNamed <IOutputController>(entity.Key) : new NullOutputController();
IPipeline pipeline = new DefaultPipeline(outputController, context);
// TODO: rely on IInputProvider's Read method instead (after every provider has one)
pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IRead)) ? ctx.ResolveNamed <IRead>(entity.Key) : null);
pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IInputProvider)) ? ctx.ResolveNamed <IInputProvider>(entity.Key) : null);
// transforms
if (!process.ReadOnly)
{
pipeline.Register(new SetSystemFields(new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity)));
}
pipeline.Register(new IncrementTransform(context));
pipeline.Register(new LogTransform(context));
pipeline.Register(new DefaultTransform(context, context.GetAllEntityFields().Where(f => !f.System)));
pipeline.Register(TransformFactory.GetTransforms(ctx, context, entity.GetAllFields().Where(f => f.Transforms.Any())));
if (!process.ReadOnly)
{
pipeline.Register(new StringTruncateTransfom(new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity)));
}
pipeline.Register(new LogTransform(context));
// writer, TODO: rely on IOutputProvider instead
pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IWrite)) ? ctx.ResolveNamed <IWrite>(entity.Key) : null);
pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IOutputProvider)) ? ctx.ResolveNamed <IOutputProvider>(entity.Key) : null);
// updater
pipeline.Register(process.ReadOnly || !ctx.IsRegisteredWithName(entity.Key, typeof(IUpdate)) ? new NullUpdater() : ctx.ResolveNamed <IUpdate>(entity.Key));
return(pipeline);
}).Named <IPipeline>(entity.Key);
}
// process pipeline
builder.Register(ctx => {
var calc = process.ToCalculatedFieldsProcess();
var entity = calc.Entities.First();
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity);
context.Debug(() => $"Registering {process.Pipeline} pipeline.");
var outputController = ctx.IsRegistered <IOutputController>() ? ctx.Resolve <IOutputController>() : new NullOutputController();
IPipeline pipeline = new DefaultPipeline(outputController, context);
// no updater necessary
pipeline.Register(new NullUpdater(context, false));
if (!process.CalculatedFields.Any())
{
pipeline.Register(new NullReader(context, false));
pipeline.Register(new NullWriter(context, false));
return(pipeline);
}
// register transforms
pipeline.Register(new IncrementTransform(context));
pipeline.Register(new LogTransform(context));
pipeline.Register(new DefaultTransform(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity), entity.CalculatedFields));
pipeline.Register(TransformFactory.GetTransforms(ctx, context, entity.CalculatedFields));
pipeline.Register(new StringTruncateTransfom(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity)));
pipeline.Register(new NullReader(context));
pipeline.Register(new NullWriter(context));
return(pipeline);
}).As <IPipeline>();
// process controller
builder.Register <IProcessController>(ctx => {
var pipelines = new List <IPipeline>();
// entity-level pipelines
foreach (var entity in process.Entities)
{
var pipeline = ctx.ResolveNamed <IPipeline>(entity.Key);
pipelines.Add(pipeline);
if (entity.Delete && process.Mode != "init")
{
pipeline.Register(ctx.ResolveNamed <IEntityDeleteHandler>(entity.Key));
}
}
// process-level pipeline for process level calculated fields
if (ctx.IsRegistered <IPipeline>())
{
pipelines.Add(ctx.Resolve <IPipeline>());
}
var context = ctx.Resolve <IContext>();
var controller = new ProcessController(pipelines, context);
// actions
foreach (var action in process.Actions.Where(a => a.GetModes().Any(m => m == process.Mode || m == "*")))
{
if (action.Before)
{
controller.PreActions.Add(ctx.ResolveNamed <IAction>(action.Key));
}
if (action.After)
{
controller.PostActions.Add(ctx.ResolveNamed <IAction>(action.Key));
}
}
return(controller);
}).As <IProcessController>();
var build = builder.Build();
return(build.BeginLifetimeScope());
}
protected override void Load(ContainerBuilder builder)
{
var loadContext = new PipelineContext(_logger, _process);
// set properties for plugins
builder.Properties["ShortHand"] = _shortHand;
builder.Properties["Methods"] = _methods;
builder.Properties["Process"] = _process;
// new method so transform author can define shorthand signature(s)
RegisterTransform(builder, (ctx, c) => new AbsTransform(c), new AbsTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new SliceTransform(c), new SliceTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new FormatTransform(c), new FormatTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new AddTransform(c), new AddTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new EqualsTransform(c), new EqualsTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new CeilingTransform(c), new CeilingTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new CoalesceTransform(c), new CoalesceTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new ConcatTransform(c), new ConcatTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new ConnectionTransform(c), new ConnectionTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new ConvertTransform(c), new ConvertTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new CopyTransform(c), new CopyTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new DateDiffTransform(c), new DateDiffTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new DatePartTransform(c), new DatePartTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new DecompressTransform(c), new DecompressTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new CompressTransform(c), new CompressTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new FileExtTransform(c), new FileExtTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new FileNameTransform(c), new FileNameTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new FilePathTransform(c), new FilePathTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new SplitTransform(c), new SplitTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new LastDayTransform(c), new LastDayTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new LastTransform(c), new LastTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new FirstTransform(c), new FirstTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new FloorTransform(c), new FloorTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new FormatXmlTransform(c), new FormatXmlTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new FormatPhoneTransform(c), new FormatPhoneTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new HashcodeTransform(c), new HashcodeTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new DecodeTransform(c), new DecodeTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new HtmlEncodeTransform(c), new HtmlEncodeTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new InsertTransform(c), new InsertTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new InvertTransform(c), new InvertTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new JoinTransform(c), new JoinTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new LeftTransform(c), new LeftTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new RightTransform(c), new RightTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new ToLowerTransform(c), new ToLowerTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new MapTransform(c), new MapTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new RegexMatchTransform(c), new RegexMatchTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new RegexMatchingTransform(c), new RegexMatchingTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new MultiplyTransform(c), new MultiplyTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new NextDayTransform(c), new NextDayTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new UtcNowTransform(c), new UtcNowTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new RowNumberTransform(c), new RowNumberTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new TimeZoneTransform(c), new TimeZoneTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new TimeZoneOffsetTransform(c), new TimeZoneOffsetTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new OppositeTransform(c), new OppositeTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new DateAddTransform(c), new DateAddTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new TimeAheadTransform(c), new TimeAheadTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new TimeAgoTransform(c), new TimeAgoTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new PadLeftTransform(c), new PadLeftTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new PadRightTransform(c), new PadRightTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new RegexReplaceTransform(c), new RegexReplaceTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new RemoveTransform(c), new RemoveTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new ReplaceTransform(c), new ReplaceTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new RoundTransform(c), new RoundTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new RoundToTransform(c), new RoundToTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new RoundUpToTransform(c), new RoundUpToTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new RoundDownToTransform(c), new RoundDownToTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new SplitLengthTransform(c), new SplitLengthTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new SubStringTransform(c), new SubStringTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new TagTransform(c), new TagTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new ToStringTransform(c), new ToStringTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new ToTimeTransform(c), new ToTimeTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new ToYesNoTransform(c), new ToYesNoTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new TrimTransform(c), new TrimTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new TrimStartTransform(c), new TrimStartTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new TrimEndTransform(c), new TrimEndTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new LineTransform(c), new LineTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new VelocityTransform(c, ctx.Resolve <IReader>()), new VelocityTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new ToUpperTransform(c), new ToUpperTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new XPathTransform(c), new XPathTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new IIfTransform(c), new IIfTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new AppendTransform(c), new AppendTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new PrependTransform(c), new PrependTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new GeohashEncodeTransform(c), new GeohashEncodeTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new GeohashNeighborTransform(c), new GeohashNeighborTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new CommonPrefixTransform(c), new CommonPrefixTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new CommonPrefixesTransform(c), new CommonPrefixesTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new DistanceTransform(c), new DistanceTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new SlugifyTransform(c), new SlugifyTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new DateMathTransform(c), new DateMathTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new WebTransform(c), new WebTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new UrlEncodeTransform(c), new UrlEncodeTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new DistinctTransform(c), new DistinctTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new RegexMatchCountTransform(c), new RegexMatchCountTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new CondenseTransform(c), new CondenseTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new RandomTransform(c), new RandomTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new IdentityTransform(c), new IdentityTransform().GetSignatures());
// row filtering
RegisterTransform(builder, (ctx, c) => new FilterTransform(FilterType.Include, c), new FilterTransform(FilterType.Include).GetSignatures());
RegisterTransform(builder, (ctx, c) => new FilterTransform(FilterType.Exclude, c), new FilterTransform(FilterType.Exclude).GetSignatures());
// field producing
RegisterTransform(builder, (ctx, c) => new FromSplitTransform(c), new FromSplitTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new FromRegexTransform(c), new FromRegexTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new FromLengthsTransform(c), new FromLengthsTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new FromJsonTransform(c, o => JsonConvert.SerializeObject(o, Formatting.None)), new FromJsonTransform().GetSignatures());
// return true or false transforms
RegisterTransform(builder, (ctx, c) => new AnyTransform(c), new AnyTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new StartsWithTransform(c), new StartsWithTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new EndsWithTransform(c), new EndsWithTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new InTransform(c), new InTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new ContainsTransform(c), new ContainsTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new IsTransform(c), new IsTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new EqualsTransform(c), new EqualsTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new IsEmptyTransform(c), new IsEmptyTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new IsDefaultTransform(c), new IsDefaultTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new IsNumericTransform(c), new IsNumericTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new RegexIsMatchTransform(c), new RegexIsMatchTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new IsDaylightSavingsTransform(c), new IsDaylightSavingsTransform().GetSignatures());
RegisterTransform(builder, (ctx, c) => new ToBoolTransform(c), new ToBoolTransform().GetSignatures());
// uncategorized
RegisterTransform(builder, (ctx, c) => new LogTransform(c), new LogTransform().GetSignatures());
// xml
RegisterTransform(builder, (ctx, c) => c.Operation.Mode == "all" || c.Field.Engine != "auto" ?
new Transforms.Xml.FromXmlTransform(ctx.ResolveNamed <IRowFactory>(c.Entity.Key, new NamedParameter("capacity", c.GetAllEntityFields().Count())), c) :
new Transforms.FromXmlTransform(c) as ITransform, new[] { new OperationSignature("fromxml") }
);
RegisterTransform(builder, (ctx, c) => new JavascriptTransform(new ChakraCoreJsEngineFactory(), ctx.Resolve <IReader>(), c), new JavascriptTransform(null, null).GetSignatures());
var pluginsFolder = Path.Combine(AssemblyDirectory, "plugins");
if (Directory.Exists(pluginsFolder))
{
var assemblies = new List <Assembly>();
foreach (var file in Directory.GetFiles(pluginsFolder, "Transformalize.Transform.*.Autofac.dll", SearchOption.TopDirectoryOnly))
{
var info = new FileInfo(file);
var name = info.Name.ToLower().Split('.').FirstOrDefault(f => f != "dll" && f != "transformalize" && f != "transform" && f != "autofac");
loadContext.Debug(() => $"Loading {name} transform(s)");
var assembly = Assembly.LoadFile(new FileInfo(file).FullName);
assemblies.Add(assembly);
}
if (assemblies.Any())
{
builder.RegisterAssemblyModules(assemblies.ToArray());
}
}
// register the short hand
builder.Register((c, p) => _shortHand).Named <ShorthandRoot>(FieldsName).InstancePerLifetimeScope();
builder.Register((c, p) => _shortHand).Named <ShorthandRoot>(ParametersName).InstancePerLifetimeScope();
builder.Register((c, p) => new ShorthandCustomizer(c.ResolveNamed <ShorthandRoot>(FieldsName), new[] { "fields", "calculated-fields" }, "t", "transforms", "method")).Named <IDependency>(FieldsName).InstancePerLifetimeScope();
builder.Register((c, p) => new ShorthandCustomizer(c.ResolveNamed <ShorthandRoot>(ParametersName), new[] { "parameters" }, "t", "transforms", "method")).Named <IDependency>(ParametersName).InstancePerLifetimeScope();
}
public ILifetimeScope CreateScope(Process process, IPipelineLogger logger)
{
var builder = new ContainerBuilder();
// the modules below rely on this process being there
builder.Properties["Process"] = process;
builder.Register(ctx => process).As <Process>();
builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance();
// register short-hand for t attribute, allowing for additional transforms
var tm = new TransformModule(process, _methods, _shortHand, logger)
{
Plugins = false
};
// adding additional transforms here
tm.AddTransform(new TransformHolder((c) => new UsernameTransform(_httpContext, c), new UsernameTransform().GetSignatures()));
tm.AddTransform(new TransformHolder((c) => new UserIdTransform(_httpContext, _userService, c), new UserIdTransform().GetSignatures()));
tm.AddTransform(new TransformHolder((c) => new UserEmailTransform(_httpContext, _userService, c), new UserEmailTransform().GetSignatures()));
tm.AddTransform(new TransformHolder((c) => new OrchardRazorTransform(c, _memoryCache, _signal), new OrchardRazorTransform().GetSignatures()));
tm.AddTransform(new TransformHolder((c) => new OrchardFluidTransform(c, _memoryCache, _signal), new OrchardFluidTransform().GetSignatures()));
tm.AddTransform(new TransformHolder((c) => new OrchardJintTransform(c, new DefaultReader(new FileReader(), new WebReader()), _memoryCache, _signal), new OrchardJintTransform().GetSignatures()));
tm.AddTransform(new TransformHolder((c) => new ToLocalTimeTransform(c, _clock, _localClock), new ToLocalTimeTransform().GetSignatures()));
tm.AddTransform(new TransformHolder((c) => new GetEncodedUrlTransform(_httpContext, c), new GetEncodedUrlTransform().GetSignatures()));
tm.AddTransform(new TransformHolder((c) => new GetDisplayUrlTransform(_httpContext, c), new GetDisplayUrlTransform().GetSignatures()));
tm.AddTransform(new TransformHolder((c) => new OrchardTimeZoneTransform(c), new OrchardTimeZoneTransform().GetSignatures()));
tm.AddTransform(new TransformHolder((c) => new GeocodeTransform(c), new GeocodeTransform().GetSignatures()));
tm.AddTransform(new TransformHolder((c) => new PlaceTransform(c), new PlaceTransform().GetSignatures()));
tm.AddTransform(new TransformHolder((c) => new FilePartTransform(c, _fileService), new FilePartTransform().GetSignatures()));
builder.RegisterModule(tm);
// register short-hand for v attribute, allowing for additional validators
var vm = new ValidateModule(process, _methods, _shortHand, logger)
{
Plugins = false
};
// adding additional validators here
builder.RegisterModule(vm);
// using custom internal module that does not handle the nested transformalize actions
builder.RegisterModule(new OrchardInternalModule(process));
// handling nested transformalize actions here instead
foreach (var action in process.Actions.Where(a => a.GetModes().Any(m => m == process.Mode || m == "*")))
{
if (action.Type == "tfl")
{
builder.Register <IAction>(ctx => {
return(new PipelineAction(action, _serviceProvider));
}).Named <IAction>(action.Key);
}
}
// register providers
var providers = new HashSet <string>(process.Connections.Select(c => c.Provider));
// relational databases
builder.RegisterModule(new AdoProviderModule());
if (providers.Contains("sqlserver"))
{
builder.RegisterModule(new SqlServerModule()
{
ConnectionFactory = (c) => new ProfiledConnectionFactory(new SqlServerConnectionFactory(c))
});
}
if (providers.Contains("postgresql"))
{
builder.RegisterModule(new PostgreSqlModule()
{
ConnectionFactory = (c) => new ProfiledConnectionFactory(new PostgreSqlConnectionFactory(c))
});
}
if (providers.Contains("sqlite"))
{
builder.RegisterModule(new SqliteModule()
{
ConnectionFactory = (c) => new ProfiledConnectionFactory(new SqliteConnectionFactory(c))
});
}
if (providers.Contains("mysql"))
{
builder.RegisterModule(new MySqlModule()
{
ConnectionFactory = (c) => new ProfiledConnectionFactory(new MySqlConnectionFactory(c))
});
}
// search engines
if (providers.Contains("elasticsearch"))
{
builder.RegisterModule(new ElasticsearchModule());
}
// solr
// lucene
// importing, exporting
var stream = _httpContext.HttpContext.Response.Body;
if (providers.Contains("file"))
{
builder.RegisterModule(new CsvHelperProviderModule(stream));
}
if (providers.Contains("excel"))
{
builder.RegisterModule(new OrchardExcelModule());
}
// exporting
if (providers.Contains("json"))
{
builder.RegisterModule(new JsonProviderModule(stream)
{
UseAsyncMethods = true
});
}
if (providers.Contains("geojson"))
{
builder.RegisterModule(new GeoJsonProviderModule(stream)
{
UseAsyncMethods = true
});
}
// misc
if (providers.Contains("bogus"))
{
builder.RegisterModule(new BogusModule());
}
if (providers.Contains("log") || process.Actions.Any(a => a.Type == "log"))
{
builder.RegisterModule(new OrchardLogModule(process));
}
if (providers.Contains("mail"))
{
builder.RegisterModule(new MailModule());
}
if (providers.Contains("aws"))
{
var services = new HashSet <string>(process.Connections.Where(c => c.Provider == "aws").Select(c => c.Service));
if (services.Contains("logs"))
{
builder.RegisterModule(new AwsCloudWatchProviderModule());
}
if (services.Contains("connect"))
{
builder.RegisterModule(new AmazonConnectProviderModule());
}
}
// transform and validation modules need these properties
builder.Properties["ShortHand"] = _shortHand;
builder.Properties["Methods"] = _methods;
// register transform modules here
builder.RegisterModule(new JsonTransformModule());
builder.RegisterModule(new HumanizeModule());
builder.RegisterModule(new FileModule());
builder.RegisterModule(new AdoTransformModule());
builder.RegisterModule(new LambdaParserModule());
builder.RegisterModule(new AwsTransformModule());
// register validator modules here
builder.RegisterModule(new JintValidateModule());
// process context
builder.Register <IContext>((ctx, p) => new PipelineContext(logger, process)).As <IContext>();
// process output context
builder.Register(ctx => {
var context = ctx.Resolve <IContext>();
return(new OutputContext(context));
}).As <OutputContext>();
// connection and process level output context
var formWriter = false;
foreach (var connection in process.Connections)
{
builder.Register(ctx => new ConnectionContext(ctx.Resolve <IContext>(), connection)).Named <IConnectionContext>(connection.Key);
// there can only be one form writer
if (!formWriter && connection.Table != "[default]" && connection.Provider != "[default]")
{
builder.Register(ctx => {
var context = ctx.ResolveNamed <IConnectionContext>(connection.Key);
var connectionFactory = ctx.ResolveNamed <IConnectionFactory>(connection.Key);
return(new AdoFormCommandWriter(context, connectionFactory));
}).As <AdoFormCommandWriter>();
formWriter = true;
}
if (connection.Name != process.Output)
{
continue;
}
// register output for connection
builder.Register(ctx => {
var context = ctx.ResolveNamed <IConnectionContext>(connection.Key);
return(new OutputContext(context));
}).Named <OutputContext>(connection.Key);
}
// entity context and rowFactory
foreach (var entity in process.Entities)
{
builder.Register <IContext>((ctx, p) => new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity)).Named <IContext>(entity.Key);
builder.Register(ctx => {
var context = ctx.ResolveNamed <IContext>(entity.Key);
return(new InputContext(context));
}).Named <InputContext>(entity.Key);
builder.Register <IRowFactory>((ctx, p) => new RowFactory(p.Named <int>("capacity"), entity.IsMaster, false)).Named <IRowFactory>(entity.Key);
builder.Register(ctx => {
var context = ctx.ResolveNamed <IContext>(entity.Key);
return(new OutputContext(context));
}).Named <OutputContext>(entity.Key);
var connection = process.Connections.First(c => c.Name == entity.Input);
builder.Register(ctx => new ConnectionContext(ctx.Resolve <IContext>(), connection)).Named <IConnectionContext>(entity.Key);
}
// entity pipelines
foreach (var entity in process.Entities)
{
builder.Register(ctx => {
var output = process.GetOutputConnection();
var context = ctx.ResolveNamed <IContext>(entity.Key);
var outputController = ctx.IsRegisteredWithName <IOutputController>(entity.Key) ? ctx.ResolveNamed <IOutputController>(entity.Key) : new NullOutputController();
var pipeline = new DefaultPipeline(outputController, context);
if (entity.IsMaster)
{
if (GetReaderAlternate != null)
{
// Using an alternate reader (e.g. a ParameterRowReader that reads parameters into a row)
var input = ctx.ResolveNamed <InputContext>(entity.Key);
var rowFactory = ctx.ResolveNamed <IRowFactory>(entity.Key, new NamedParameter("capacity", input.RowCapacity));
pipeline.Register(GetReaderAlternate(input, rowFactory));
}
else if (GetReaderDecorator != null && ctx.IsRegisteredWithName(entity.Key, typeof(IRead)))
{
// Decorating the normally registered reader (e.g. ParameterRowReader updating a row from an AdoReader)
// TODO: Support IInputProvider instead of just IRead
var input = ctx.ResolveNamed <InputContext>(entity.Key);
var rowFactory = ctx.ResolveNamed <IRowFactory>(entity.Key, new NamedParameter("capacity", input.RowCapacity));
pipeline.Register(GetReaderDecorator(ctx.ResolveNamed <IRead>(entity.Key), input, rowFactory));
}
else
{
pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IRead)) ? ctx.ResolveNamed <IRead>(entity.Key) : null);
pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IInputProvider)) ? ctx.ResolveNamed <IInputProvider>(entity.Key) : null);
}
}
else
{
// TODO: rely on IInputProvider's Read method instead (after every provider has one)
pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IRead)) ? ctx.ResolveNamed <IRead>(entity.Key) : null);
pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IInputProvider)) ? ctx.ResolveNamed <IInputProvider>(entity.Key) : null);
}
// transforms
if (!process.ReadOnly)
{
pipeline.Register(new SetSystemFields(new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity)));
}
pipeline.Register(new IncrementTransform(context));
pipeline.Register(new DefaultTransform(context, context.GetAllEntityFields().Where(f => !f.System)));
pipeline.Register(TransformFactory.GetTransforms(ctx, context, entity.GetAllFields().Where(f => f.Transforms.Any())));
pipeline.Register(ValidateFactory.GetValidators(ctx, context, entity.GetAllFields().Where(f => f.Validators.Any())));
if (!process.ReadOnly && !output.Provider.In("internal", "log"))
{
pipeline.Register(new StringTruncateTransfom(new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity)));
}
pipeline.Register(new LogTransform(context));
// writer, TODO: rely on IOutputProvider instead
pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IWrite)) ? ctx.ResolveNamed <IWrite>(entity.Key) : null);
pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IOutputProvider)) ? ctx.ResolveNamed <IOutputProvider>(entity.Key) : null);
// updater
pipeline.Register(process.ReadOnly || !ctx.IsRegisteredWithName(entity.Key, typeof(IUpdate)) ? new NullUpdater() : ctx.ResolveNamed <IUpdate>(entity.Key));
return(pipeline);
}).Named <IPipeline>(entity.Key);
}
if (process.Entities.Count > 1 && process.Relationships.Any())
{
// process pipeline
builder.Register(ctx => {
var calc = process.ToCalculatedFieldsProcess();
var entity = calc.Entities.First();
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity);
var outputContext = new OutputContext(context);
context.Debug(() => $"Registering {process.Pipeline} pipeline.");
var outputController = ctx.IsRegistered <IOutputController>() ? ctx.Resolve <IOutputController>() : new NullOutputController();
var pipeline = new DefaultPipeline(outputController, context);
// no updater necessary
pipeline.Register(new NullUpdater(context, false));
if (!process.CalculatedFields.Any())
{
pipeline.Register(new NullReader(context, false));
pipeline.Register(new NullWriter(context, false));
return(pipeline);
}
// register transforms
pipeline.Register(new IncrementTransform(context));
pipeline.Register(new LogTransform(context));
pipeline.Register(new DefaultTransform(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity), entity.CalculatedFields));
pipeline.Register(TransformFactory.GetTransforms(ctx, context, entity.CalculatedFields));
pipeline.Register(ValidateFactory.GetValidators(ctx, context, entity.GetAllFields().Where(f => f.Validators.Any())));
pipeline.Register(new StringTruncateTransfom(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity)));
// register input and output
pipeline.Register(ctx.IsRegistered <IRead>() ? ctx.Resolve <IRead>() : new NullReader(context));
pipeline.Register(ctx.IsRegistered <IWrite>() ? ctx.Resolve <IWrite>() : new NullWriter(context));
if (outputContext.Connection.Provider == "sqlserver")
{
pipeline.Register(new MinDateTransform(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity), new DateTime(1753, 1, 1)));
}
return(pipeline);
}).As <IPipeline>();
}
// process controller
builder.Register <IProcessController>(ctx => {
var pipelines = new List <IPipeline>();
// entity-level pipelines
foreach (var entity in process.Entities)
{
var pipeline = ctx.ResolveNamed <IPipeline>(entity.Key);
pipelines.Add(pipeline);
if (entity.Delete && process.Mode != "init")
{
pipeline.Register(ctx.ResolveNamed <IEntityDeleteHandler>(entity.Key));
}
}
// process-level pipeline for process level calculated fields
if (ctx.IsRegistered <IPipeline>())
{
pipelines.Add(ctx.Resolve <IPipeline>());
}
var context = ctx.Resolve <IContext>();
var controller = new ProcessController(pipelines, context);
// output initialization
if (process.Mode == "init" && ctx.IsRegistered <IInitializer>())
{
controller.PreActions.Add(ctx.Resolve <IInitializer>());
}
// flatten(ing) is first post-action
var output = process.GetOutputConnection();
var isAdo = _adoProviders.Contains(output.Provider);
if (process.Flatten && isAdo)
{
if (ctx.IsRegisteredWithName <IAction>(output.Key))
{
controller.PostActions.Add(ctx.ResolveNamed <IAction>(output.Key));
}
else
{
context.Error($"Could not find ADO Flatten Action for provider {output.Provider}.");
}
}
// actions
foreach (var action in process.Actions.Where(a => a.GetModes().Any(m => m == process.Mode || m == "*")))
{
if (ctx.IsRegisteredWithName <IAction>(action.Key))
{
if (action.Before)
{
controller.PreActions.Add(ctx.ResolveNamed <IAction>(action.Key));
}
if (action.After)
{
controller.PostActions.Add(ctx.ResolveNamed <IAction>(action.Key));
}
}
else
{
if (action.Type != "internal")
{
_logger.Warn(() => $"The action {action.Name} with type {action.Type} isn't registered.");
}
}
}
foreach (var map in process.Maps.Where(m => !string.IsNullOrEmpty(m.Query)))
{
controller.PreActions.Add(new MapReaderAction(context, map, ctx.ResolveNamed <IMapReader>(map.Name)));
}
return(controller);
}).As <IProcessController>();
var build = builder.Build();
return(build.BeginLifetimeScope());
}