public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) { builder.Register<IRead>(ctx => { var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity); var input = new InputContext(context, new Incrementer(context)); switch (input.Connection.Provider) { case "internal": context.Debug("Registering {0} provider", input.Connection.Provider); return new DataSetEntityReader(input); case "sqlserver": if (input.Entity.ReadSize == 0) { context.Debug("Registering {0} reader", input.Connection.Provider); return new SqlInputReader(input, input.InputFields); } context.Debug("Registering {0} batch reader", input.Connection.Provider); return new SqlInputBatchReader( input, new SqlInputReader(input, input.Entity.GetPrimaryKey()) ); default: context.Warn("Registering null reader", input.Connection.Provider); return new NullEntityReader(); } }).Named<IRead>(entity.Key); }
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) { builder.Register <IRead>(ctx => { var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity); var input = new InputContext(context, new Incrementer(context)); switch (input.Connection.Provider) { case "internal": context.Debug("Registering {0} provider", input.Connection.Provider); return(new DataSetEntityReader(input)); case "sqlserver": if (input.Entity.ReadSize == 0) { context.Debug("Registering {0} reader", input.Connection.Provider); return(new SqlInputReader(input, input.InputFields)); } context.Debug("Registering {0} batch reader", input.Connection.Provider); return(new SqlInputBatchReader( input, new SqlInputReader(input, input.Entity.GetPrimaryKey()) )); default: context.Warn("Registering null reader", input.Connection.Provider); return(new NullEntityReader()); } }).Named <IRead>(entity.Key); }
public string Render() { var p = new Dictionary <string, string>(); var l = new Cfg.Net.Loggers.MemoryLogger(); // get template _context.Debug(() => $"Reading {_template.File}"); var templateContent = _templateReader.Read(_template.File, p, l); if (l.Errors().Any()) { foreach (var error in l.Errors()) { _context.Error(error); } return(string.Empty); } try { _context.Debug(() => $"Compiling {_template.Name}."); var context = new VelocityContext(); context.Put("Process", _context.Process); foreach (var parameter in _template.Parameters) { context.Put(parameter.Name, Constants.ConversionMap[parameter.Type](parameter.Value)); } if (p.Any()) { foreach (var parameter in p) { context.Put(parameter.Key, parameter.Value); } } var sb = new StringBuilder(); using (var sw = new StringWriter(sb)) { NVelocity.App.Velocity.Evaluate(context, sw, string.Empty, templateContent); sw.Flush(); } return(sb.ToString()); } catch (Exception ex) { _context.Error($"Error parsing template {_template.Name}."); _context.Error(ex, ex.Message.Replace("{", "{{").Replace("}", "}}")); return(string.Empty); } }
public void Start() { using (var outer = ConfigurationContainer.Create(_options.Arrangement, _logger, null, _options.PlaceHolderStyle)) { var process = outer.Resolve <Configuration.Process>(); var log = new PipelineContext(_logger, process); if (process.Entities.Any(e => process.Connections.First(c => c.Name == e.Connection).Provider != Constants.DefaultSetting && !e.Fields.Any(f => f.Input))) { log.Debug(() => "Detecting schema..."); if (_schemaHelper.Help(process)) { process.Check(); } } if (_options.Mode != null && _options.Mode.ToLower() == "check") { SimplifyForOutput(process); Console.WriteLine(process.Serialize()); return; } if (_options.Mode != "default") { process.Mode = _options.Mode; } foreach (var warning in process.Warnings()) { log.Debug(() => warning); } if (process.Errors().Any()) { foreach (var error in process.Errors()) { log.Error(error); } log.Error("The configuration errors must be fixed before this job will run."); log.Logger.Clear(); return; } using (var inner = DefaultContainer.Create(process, _logger, _options.PlaceHolderStyle)) { inner.Resolve <IProcessController>().Execute(); } } }
protected override void Load(ContainerBuilder builder) { if (_process == null) { return; } // razor moved to plugin, velocity too soon... foreach (var t in _process.Templates.Where(t => t.Enabled && t.Engine != "razor")) { var template = t; builder.Register <ITemplateEngine>(ctx => { var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), _process); context.Debug(() => $"Registering {template.Engine} Engine for {t.Key}"); switch (template.Engine) { case "velocity": return(new VelocityTemplateEngine(context, template, ctx.Resolve <IReader>())); default: return(new NullTemplateEngine()); } }).Named <ITemplateEngine>(t.Key); } }
protected override void Load(ContainerBuilder builder) { new TransformBuilder(_process, builder, _methods, _shortHand, _transforms, _logger).Build(); #if PLUGINS var context = new PipelineContext(_logger, _process); builder.Properties["ShortHand"] = _shortHand; builder.Properties["Methods"] = _methods; builder.Properties["Process"] = _process; var pluginsFolder = Path.Combine(AssemblyDirectory, "plugins"); if (Directory.Exists(pluginsFolder)) { var assemblies = new List <Assembly>(); foreach (var file in Directory.GetFiles(pluginsFolder, "Transformalize.Transform.*.Autofac.dll", SearchOption.TopDirectoryOnly)) { var info = new FileInfo(file); var name = info.Name.ToLower().Split('.').FirstOrDefault(f => f != "dll" && f != "transformalize" && f != "transform" && f != "autofac"); context.Debug(() => $"Loading {name} transform(s)"); var assembly = Assembly.LoadFile(new FileInfo(file).FullName); assemblies.Add(assembly); } if (assemblies.Any()) { builder.RegisterAssemblyModules(assemblies.ToArray()); } } #endif }
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) { builder.Register<IEntityController>(ctx => { var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity); var output = new OutputContext(context, new Incrementer(context)); switch (output.Connection.Provider) { case "sqlserver": context.Debug("Registering sql server controller"); var initializer = process.Mode == "init" ? (IAction)new SqlEntityInitializer(output) : new NullInitializer(); return new SqlEntityController(output, initializer); default: context.Debug("Registering null controller"); return new NullEntityController(); } }).Named<IEntityController>(entity.Key); }
protected override void Load(ContainerBuilder builder) { var loadContext = new PipelineContext(_logger, _process); builder.Properties["ShortHand"] = _shortHand; builder.Properties["Methods"] = _methods; builder.Properties["Process"] = _process; // return true or false, validators // new style RegisterValidator(builder, (ctx, c) => new AnyValidator(c), new AnyValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new AllValidator(c), new AllValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new StartsWithValidator(c), new StartsWithValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new EndsWithValidator(c), new EndsWithValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new MapValidator(true, c), new MapValidator(inMap: true).GetSignatures()); RegisterValidator(builder, (ctx, c) => new MapValidator(false, c), new MapValidator(inMap: false).GetSignatures()); RegisterValidator(builder, (ctx, c) => new ContainsValidator(c), new ContainsValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new IsValidator(c), new IsValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new EqualsValidator(c), new EqualsValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new NotEqualValidator(c), new NotEqualValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new EmptyValidator(c), new EmptyValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new DefaultValidator(c), new DefaultValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new NumericValidator(c), new NumericValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new MatchValidator(c), new MatchValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new RequiredValidator(c), new RequiredValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new LengthValidator(c), new LengthValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new MinLengthValidator(c), new MinLengthValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new MaxLengthValidator(c), new MaxLengthValidator().GetSignatures()); RegisterValidator(builder, (ctx, c) => new CompareValidator("min", c), new CompareValidator("min").GetSignatures()); RegisterValidator(builder, (ctx, c) => new CompareValidator("max", c), new CompareValidator("max").GetSignatures()); RegisterValidator(builder, (ctx, c) => new RegularExpressionValidator("alphanum", "^[a-zA-Z0-9]*$", "must be alphanumeric", c), new RegularExpressionValidator("alphanum", "^[a-zA-Z0-9]*$", "must be alphanumeric").GetSignatures()); var pluginsFolder = Path.Combine(AssemblyDirectory, "plugins"); if (Directory.Exists(pluginsFolder)) { var assemblies = new List <Assembly>(); foreach (var file in Directory.GetFiles(pluginsFolder, "Transformalize.Validate.*.Autofac.dll", SearchOption.TopDirectoryOnly)) { var info = new FileInfo(file); var name = info.Name.ToLower().Split('.').FirstOrDefault(f => f != "dll" && f != "transformalize" && f != "validate" && f != "autofac"); loadContext.Debug(() => $"Loading {name} validator(s)"); var assembly = Assembly.LoadFile(new FileInfo(file).FullName); assemblies.Add(assembly); } if (assemblies.Any()) { builder.RegisterAssemblyModules(assemblies.ToArray()); } } // register the short hand builder.Register((c, p) => _shortHand).Named <ShorthandRoot>(Name).InstancePerLifetimeScope(); builder.Register((c, p) => new ShorthandCustomizer(c.ResolveNamed <ShorthandRoot>(Name), new[] { "fields", "calculated-fields" }, "v", "validators", "method")).Named <IDependency>(Name).InstancePerLifetimeScope(); }
public string Render() { var p = new Dictionary <string, string>(); var l = new Cfg.Net.Loggers.MemoryLogger(); // get template _context.Debug(() => $"Reading {_template.File}"); var templateContent = _templateReader.Read(_template.File, p, l); if (l.Errors().Any()) { foreach (var error in l.Errors()) { _context.Error(error); } return(string.Empty); } // get parameters (other than process) var parameters = new ExpandoObject(); foreach (var parameter in _template.Parameters) { ((IDictionary <string, object>)parameters).Add(parameter.Name, parameter.Value); } if (p.Any()) { foreach (var parameter in p) { ((IDictionary <string, object>)parameters)[parameter.Key] = parameter.Value; } } try { _context.Debug(() => $"Compiling {_template.Name}."); return(_service.RunCompile(templateContent, _template.Name, null, new { _context.Process, Parameters = parameters })); } catch (Exception ex) { _context.Error($"Error parsing template {_template.Name}."); _context.Error(ex, ex.Message.Replace("{", "{{").Replace("}", "}}")); return(string.Empty); } }
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) { builder.Register <IEntityController>(ctx => { var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity); var output = new OutputContext(context, new Incrementer(context)); switch (output.Connection.Provider) { case "sqlserver": context.Debug("Registering sql server controller"); var initializer = process.Mode == "init" ? (IAction) new SqlEntityInitializer(output) : new NullInitializer(); return(new SqlEntityController(output, initializer)); default: context.Debug("Registering null controller"); return(new NullEntityController()); } }).Named <IEntityController>(entity.Key); }
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) { var type = process.Pipeline == "defer" ? entity.Pipeline : process.Pipeline; builder.Register <IPipeline>((ctx) => { var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity); IPipeline pipeline; switch (type) { case "parallel.linq": context.Debug("Registering {0} pipeline.", type); pipeline = new ParallelPipeline(new DefaultPipeline(ctx.ResolveNamed <IEntityController>(entity.Key), context)); break; default: context.Debug("Registering linq pipeline.", type); pipeline = new DefaultPipeline(ctx.ResolveNamed <IEntityController>(entity.Key), context); break; } var provider = process.Connections.First(c => c.Name == "output").Provider; // extract pipeline.Register(ctx.ResolveNamed <IRead>(entity.Key)); // transform pipeline.Register(new DefaultTransform(context, context.GetAllEntityFields())); pipeline.Register(new TflHashCodeTransform(context)); pipeline.Register(TransformFactory.GetTransforms(ctx, process, entity, entity.GetAllFields().Where(f => f.Transforms.Any()))); pipeline.Register(new StringTruncateTransfom(context)); if (provider == "sqlserver") { pipeline.Register(new MinDateTransform(context, new DateTime(1753, 1, 1))); } //load pipeline.Register(ctx.ResolveNamed <IWrite>(entity.Key)); pipeline.Register(ctx.ResolveNamed <IUpdate>(entity.Key)); return(pipeline); }).Named <IPipeline>(entity.Key); }
protected override void Load(ContainerBuilder builder) { if (!builder.Properties.ContainsKey("Process")) { return; } var process = (Process)builder.Properties["Process"]; // template engines for template actions creating output with an action foreach (var t in process.Templates.Where(t => t.Enabled && t.Engine == Razor)) { var template = t; builder.Register <ITemplateEngine>(ctx => { var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process); context.Debug(() => $"Registering {template.Engine} Engine for {t.Key}"); return(new RazorTemplateEngine(context, template, ctx.Resolve <IReader>())); }).Named <ITemplateEngine>(t.Key); } // connections, no schema for razor foreach (var connection in process.Connections.Where(c => c.Provider == Razor)) { builder.Register <ISchemaReader>(ctx => new NullSchemaReader()).Named <ISchemaReader>(connection.Key); } // Entity input foreach (var entity in process.Entities.Where(e => process.Connections.First(c => c.Name == e.Input).Provider == Razor)) { // no input version detector builder.RegisterType <NullInputProvider>().Named <IInputProvider>(entity.Key); // no input reader builder.Register <IRead>(ctx => { var input = ctx.ResolveNamed <InputContext>(entity.Key); return(new NullReader(input, false)); }).Named <IRead>(entity.Key); } if (process.GetOutputConnection().Provider == Razor) { // PROCESS OUTPUT CONTROLLER builder.Register <IOutputController>(ctx => new NullOutputController()).As <IOutputController>(); foreach (var entity in process.Entities) { builder.Register <IOutputController>(ctx => new NullOutputController()).Named <IOutputController>(entity.Key); // ENTITY WRITER builder.Register <IWrite>(ctx => new RazorWriter(ctx.ResolveNamed <OutputContext>(entity.Key), new DefaultReader(new FileReader(), new WebReader()))).Named <IWrite>(entity.Key); } } }
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) { var type = process.Pipeline == "defer" ? entity.Pipeline : process.Pipeline; builder.Register<IPipeline>((ctx) => { var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity); IPipeline pipeline; switch (type) { case "parallel.linq": context.Debug("Registering {0} pipeline.", type); pipeline = new ParallelPipeline(new DefaultPipeline(ctx.ResolveNamed<IEntityController>(entity.Key), context)); break; default: context.Debug("Registering linq pipeline.", type); pipeline = new DefaultPipeline(ctx.ResolveNamed<IEntityController>(entity.Key), context); break; } var provider = process.Connections.First(c => c.Name == "output").Provider; // extract pipeline.Register(ctx.ResolveNamed<IRead>(entity.Key)); // transform pipeline.Register(new DefaultTransform(context, context.GetAllEntityFields())); pipeline.Register(new TflHashCodeTransform(context)); pipeline.Register(TransformFactory.GetTransforms(ctx,process, entity, entity.GetAllFields().Where(f=>f.Transforms.Any()))); pipeline.Register(new StringTruncateTransfom(context)); if (provider == "sqlserver") { pipeline.Register(new MinDateTransform(context, new DateTime(1753, 1, 1))); } //load pipeline.Register(ctx.ResolveNamed<IWrite>(entity.Key)); pipeline.Register(ctx.ResolveNamed<IUpdate>(entity.Key)); return pipeline; }).Named<IPipeline>(entity.Key); }
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) { //master updater builder.Register<IUpdate>((ctx) => { var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity); var incrementer = new Incrementer(context); var output = new OutputContext(context, incrementer); switch (output.Connection.Provider) { case "sqlserver": context.Debug("Registering {0} master updater", output.Connection.Provider); return new SqlMasterUpdater(output); default: context.Warn("Registering null updater"); return new NullMasterUpdater(); } }).Named<IUpdate>(entity.Key); }
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) { builder.Register <IWrite>((ctx) => { var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity); var incrementer = new Incrementer(context); var output = new OutputContext(context, incrementer); switch (output.Connection.Provider) { case "sqlserver": context.Debug("Registering Sql Server writer"); return(new SqlEntityBulkInserter(output)); default: context.Warn("Registering null writer", output.Connection.Provider); return(new NullWriter(context)); } }).Named <IWrite>(entity.Key); }
protected override void Load(ContainerBuilder builder) { if (_process == null) { return; } foreach (var t in _process.Templates.Where(t => t.Enabled)) { var template = t; builder.Register <ITemplateEngine>(ctx => { var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), _process); context.Debug(() => string.Format("Registering {0} Engine for {1}", template.Engine, t.Key)); switch (template.Engine) { case "razor": return(new OrchardRazorTemplateEngine(context, _templateProcessor, template, ctx.Resolve <IReader>())); default: return(new NullTemplateEngine()); } }).Named <ITemplateEngine>(t.Key); } }
public static ILifetimeScope Create(Process process, IPipelineLogger logger, string placeHolderStyle) { var loadContext = new PipelineContext(logger, process); if (process.OutputIsConsole()) { logger.SuppressConsole(); } var builder = new ContainerBuilder(); builder.Properties["Process"] = process; builder.Register(ctx => placeHolderStyle).Named <string>("placeHolderStyle"); builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance(); /* this stuff is loaded (again) because tfl actions can create processes, which will need short-hand to expand configuration in advance */ builder.RegisterCallback(new TransformModule(process, logger).Configure); builder.RegisterCallback(new ValidateModule(process, logger).Configure); builder.RegisterCallback(new RootModule().Configure); builder.RegisterCallback(new ContextModule(process).Configure); // provider loading section var providers = new HashSet <string>(process.Connections.Select(c => c.Provider).Distinct(), StringComparer.OrdinalIgnoreCase); builder.RegisterCallback(new InternalModule(process).Configure); builder.RegisterCallback(new FileModule(process).Configure); if (providers.Contains("console")) { builder.RegisterCallback(new ConsoleModule(process).Configure); } if (providers.Contains("kml")) { builder.RegisterCallback(new KmlModule(process).Configure); } if (providers.Contains("filesystem")) { builder.RegisterCallback(new FileSystemModule(process).Configure); } var pluginsFolder = Path.Combine(AssemblyDirectory, "plugins"); if (Directory.Exists(pluginsFolder)) { var assemblies = new List <Assembly>(); var files = Directory.GetFiles(pluginsFolder, "Transformalize.Provider.*.Autofac.dll", SearchOption.TopDirectoryOnly); foreach (var file in files) { var info = new FileInfo(file); var name = info.Name.ToLower().Split('.').FirstOrDefault(f => f != "dll" && f != "transformalize" && f != "provider" && f != "autofac"); // temporary hack if (name.StartsWith("amazonkinesis")) { name = name.Replace("amazonkinesis", string.Empty); } switch (name) { case "filehelpers" when(providers.Contains("file") || providers.Contains("folder")): loadContext.Debug(() => "Loading filehelpers provider"); assemblies.Add(Assembly.LoadFile(new FileInfo(file).FullName)); break; case "ado": loadContext.Debug(() => "Loading ADO provider"); assemblies.Add(Assembly.LoadFile(new FileInfo(file).FullName)); break; default: if (providers.Contains(name)) { loadContext.Debug(() => $"Loading {name} provider."); var assembly = Assembly.LoadFile(new FileInfo(file).FullName); assemblies.Add(assembly); } else { loadContext.Debug(() => $"Loading {name} isn't necessary for this arrangement."); } break; } } if (assemblies.Any()) { builder.RegisterAssemblyModules(assemblies.ToArray()); } } // etc builder.RegisterCallback(new EntityPipelineModule(process).Configure); builder.RegisterCallback(new ProcessPipelineModule(process).Configure); builder.RegisterCallback(new ProcessControlModule(process).Configure); return(builder.Build().BeginLifetimeScope()); }
public string Write(EntityStatus status) { /* if update occurs, outside of first run, you must update the master's * batch id to the entity's batch id (which will be higher). * Updating this indicates a change and we have to update process-level calculated columns * on the master record. * * It also keeps the master's batch id incrementing to indicate * updates for subsequent processes using this process' output as an input with TflBatchId * as a version field. */ var masterEntity = _c.Process.Entities.First(e => e.IsMaster); var masterTable = _cf.Enclose(masterEntity.OutputTableName(_c.Process.Name)); var masterAlias = masterEntity.GetExcelName(); var entityAlias = _c.Entity.GetExcelName(); var builder = new StringBuilder(); builder.AppendLine($"UPDATE {masterAlias}"); builder.Append("SET "); var setPrefix = masterAlias + "."; foreach (var field in _c.Entity.Fields.Where(f => f.KeyType.HasFlag(KeyType.Foreign))) { builder.AppendLine($"{setPrefix}{_cf.Enclose(field.FieldName())} = {entityAlias}.{_cf.Enclose(field.FieldName())},"); } builder.AppendLine($"{setPrefix}{_cf.Enclose(masterEntity.TflBatchId().FieldName())} = @TflBatchId"); builder.AppendFormat(" FROM {0} {1}", masterTable, masterAlias); var relationships = _c.Entity.RelationshipToMaster.Reverse().ToArray(); for (var r = 0; r < relationships.Length; r++) { var relationship = relationships[r]; var right = _cf.Enclose(relationship.Summary.RightEntity.OutputTableName(_c.Process.Name)); var rightEntityAlias = relationship.Summary.RightEntity.GetExcelName(); builder.AppendFormat(" INNER JOIN {0} {1} ON ( ", right, rightEntityAlias); var leftEntityAlias = relationship.Summary.LeftEntity.GetExcelName(); for (var i = 0; i < relationship.Summary.LeftFields.Count(); i++) { var leftAlias = relationship.Summary.LeftFields[i].FieldName(); var rightAlias = relationship.Summary.RightFields[i].FieldName(); var conjunction = i > 0 ? " AND " : string.Empty; builder.AppendFormat( "{0}{1}.{2} = {3}.{4}", conjunction, leftEntityAlias, _cf.Enclose(leftAlias), rightEntityAlias, _cf.Enclose(rightAlias) ); } builder.AppendLine(")"); } builder.Append("WHERE "); if (status.Modified) { builder.AppendLine($"{entityAlias}.{_cf.Enclose(_c.Entity.TflBatchId().FieldName())} = @TflBatchId "); if (status.MasterUpserted && status.HasForeignKeys) { builder.AppendLine($"OR {masterAlias}.{_cf.Enclose(masterEntity.TflBatchId().FieldName())} >= @MasterTflBatchId"); } } else if (status.MasterUpserted && status.HasForeignKeys) { builder.AppendLine($"{masterAlias}.{_cf.Enclose(masterEntity.TflBatchId().FieldName())} >= @MasterTflBatchId"); } var sql = builder.ToString(); _c.Debug(() => sql); return(sql); }
protected override void Load(ContainerBuilder builder) { var loadContext = new PipelineContext(_logger, _process); builder.Properties["ShortHand"] = _shortHand; builder.Properties["Methods"] = _methods; builder.Properties["Process"] = _process; // new method so transform author can define shorthand signature(s) RegisterTransform(builder, c => new AbsTransform(c), new AbsTransform().GetSignatures()); RegisterTransform(builder, c => new SliceTransform(c), new SliceTransform().GetSignatures()); RegisterTransform(builder, c => new FormatTransform(c), new FormatTransform().GetSignatures()); RegisterTransform(builder, c => new AddTransform(c), new AddTransform().GetSignatures()); RegisterTransform(builder, c => new EqualsTransform(c), new EqualsTransform().GetSignatures()); RegisterTransform(builder, c => new CeilingTransform(c), new CeilingTransform().GetSignatures()); RegisterTransform(builder, c => new CoalesceTransform(c), new CoalesceTransform().GetSignatures()); RegisterTransform(builder, c => new ConcatTransform(c), new ConcatTransform().GetSignatures()); RegisterTransform(builder, c => new ConnectionTransform(c), new ConnectionTransform().GetSignatures()); RegisterTransform(builder, c => new ConvertTransform(c), new ConvertTransform().GetSignatures()); RegisterTransform(builder, c => new CopyTransform(c), new CopyTransform().GetSignatures()); RegisterTransform(builder, c => new DateDiffTransform(c), new DateDiffTransform().GetSignatures()); RegisterTransform(builder, c => new DatePartTransform(c), new DatePartTransform().GetSignatures()); RegisterTransform(builder, c => new DecompressTransform(c), new DecompressTransform().GetSignatures()); RegisterTransform(builder, c => new CompressTransform(c), new CompressTransform().GetSignatures()); RegisterTransform(builder, c => new FileExtTransform(c), new FileExtTransform().GetSignatures()); RegisterTransform(builder, c => new FileNameTransform(c), new FileNameTransform().GetSignatures()); RegisterTransform(builder, c => new FilePathTransform(c), new FilePathTransform().GetSignatures()); RegisterTransform(builder, c => new SplitTransform(c), new SplitTransform().GetSignatures()); RegisterTransform(builder, c => new LastDayTransform(c), new LastDayTransform().GetSignatures()); RegisterTransform(builder, c => new LastTransform(c), new LastTransform().GetSignatures()); RegisterTransform(builder, c => new FirstTransform(c), new FirstTransform().GetSignatures()); RegisterTransform(builder, c => new FloorTransform(c), new FloorTransform().GetSignatures()); RegisterTransform(builder, c => new FormatXmlTransform(c), new FormatXmlTransform().GetSignatures()); RegisterTransform(builder, c => new FormatPhoneTransform(c), new FormatPhoneTransform().GetSignatures()); RegisterTransform(builder, c => new HashcodeTransform(c), new HashcodeTransform().GetSignatures()); RegisterTransform(builder, c => new DecodeTransform(c), new DecodeTransform().GetSignatures()); RegisterTransform(builder, c => new HtmlEncodeTransform(c), new HtmlEncodeTransform().GetSignatures()); RegisterTransform(builder, c => new InsertTransform(c), new InsertTransform().GetSignatures()); RegisterTransform(builder, c => new InvertTransform(c), new InvertTransform().GetSignatures()); RegisterTransform(builder, c => new JoinTransform(c), new JoinTransform().GetSignatures()); RegisterTransform(builder, c => new LeftTransform(c), new LeftTransform().GetSignatures()); RegisterTransform(builder, c => new RightTransform(c), new RightTransform().GetSignatures()); RegisterTransform(builder, c => new ToLowerTransform(c), new ToLowerTransform().GetSignatures()); RegisterTransform(builder, c => new MapTransform(c), new MapTransform().GetSignatures()); RegisterTransform(builder, c => new RegexMatchTransform(c), new RegexMatchTransform().GetSignatures()); RegisterTransform(builder, c => new RegexMatchingTransform(c), new RegexMatchingTransform().GetSignatures()); RegisterTransform(builder, c => new MultiplyTransform(c), new MultiplyTransform().GetSignatures()); RegisterTransform(builder, c => new NextTransform(c), new NextTransform().GetSignatures()); RegisterTransform(builder, c => new UtcNowTransform(c), new UtcNowTransform().GetSignatures()); RegisterTransform(builder, c => new RowNumberTransform(c), new RowNumberTransform().GetSignatures()); RegisterTransform(builder, c => new TimeZoneTransform(c), new TimeZoneTransform().GetSignatures()); RegisterTransform(builder, c => new TimeZoneOffsetTransform(c), new TimeZoneOffsetTransform().GetSignatures()); RegisterTransform(builder, c => new OppositeTransform(c), new OppositeTransform().GetSignatures()); RegisterTransform(builder, c => new DateAddTransform(c), new DateAddTransform().GetSignatures()); RegisterTransform(builder, c => new TimeAheadTransform(c), new TimeAheadTransform().GetSignatures()); RegisterTransform(builder, c => new TimeAgoTransform(c), new TimeAgoTransform().GetSignatures()); var pluginsFolder = Path.Combine(AssemblyDirectory, "plugins"); if (Directory.Exists(pluginsFolder)) { var assemblies = new List <Assembly>(); foreach (var file in Directory.GetFiles(pluginsFolder, "Transformalize.Transform.*.Autofac.dll", SearchOption.TopDirectoryOnly)) { var info = new FileInfo(file); var name = info.Name.ToLower().Split('.').FirstOrDefault(f => f != "dll" && f != "transformalize" && f != "transform" && f != "autofac"); loadContext.Debug(() => $"Loading {name} transform"); var assembly = Assembly.LoadFile(new FileInfo(file).FullName); assemblies.Add(assembly); } if (assemblies.Any()) { builder.RegisterAssemblyModules(assemblies.ToArray()); } } // register the short hand builder.Register((c, p) => _shortHand).Named <ShorthandRoot>(Name).InstancePerLifetimeScope(); // old method builder.Register((c, p) => new PadLeftTransform(p.Positional <IContext>(0))).Named <ITransform>("padleft"); builder.Register((c, p) => new PadRightTransform(p.Positional <IContext>(0))).Named <ITransform>("padright"); builder.Register((c, p) => new RazorTransform(p.Positional <IContext>(0))).Named <ITransform>("razor"); builder.Register((c, p) => new RegexReplaceTransform(p.Positional <IContext>(0))).Named <ITransform>("regexreplace"); builder.Register((c, p) => new RemoveTransform(p.Positional <IContext>(0))).Named <ITransform>("remove"); builder.Register((c, p) => new ReplaceTransform(p.Positional <IContext>(0))).Named <ITransform>("replace"); builder.Register((c, p) => new RoundTransform(p.Positional <IContext>(0))).Named <ITransform>("round"); builder.Register((c, p) => new RoundToTransform(p.Positional <IContext>(0), RoundTo.Nearest)).Named <ITransform>("roundto"); builder.Register((c, p) => new RoundToTransform(p.Positional <IContext>(0), RoundTo.Up)).Named <ITransform>("roundupto"); builder.Register((c, p) => new RoundToTransform(p.Positional <IContext>(0), RoundTo.Down)).Named <ITransform>("rounddownto"); builder.Register((c, p) => new SplitLengthTransform(p.Positional <IContext>(0))).Named <ITransform>("splitlength"); builder.Register((c, p) => new SubStringTransform(p.Positional <IContext>(0))).Named <ITransform>("substring"); builder.Register((c, p) => new TagTransform(p.Positional <IContext>(0))).Named <ITransform>("tag"); builder.Register((c, p) => new ToStringTransform(p.Positional <IContext>(0))).Named <ITransform>("tostring"); builder.Register((c, p) => new ToTimeTransform(p.Positional <IContext>(0))).Named <ITransform>("totime"); builder.Register((c, p) => new ToYesNoTransform(p.Positional <IContext>(0))).Named <ITransform>("toyesno"); builder.Register((c, p) => new TrimTransform(p.Positional <IContext>(0))).Named <ITransform>("trim"); builder.Register((c, p) => new TrimEndTransform(p.Positional <IContext>(0))).Named <ITransform>("trimend"); builder.Register((c, p) => new TrimStartTransform(p.Positional <IContext>(0))).Named <ITransform>("trimstart"); builder.Register((c, p) => new VelocityTransform(p.Positional <IContext>(0), c.Resolve <IReader>())).Named <ITransform>("velocity"); builder.Register((c, p) => new ToUpperTransform(p.Positional <IContext>(0))).Named <ITransform>("upper"); builder.Register((c, p) => new ToUpperTransform(p.Positional <IContext>(0))).Named <ITransform>("toupper"); builder.Register((c, p) => new XPathTransform(p.Positional <IContext>(0))).Named <ITransform>("xpath"); builder.Register((c, p) => new IIfTransform(p.Positional <IContext>(0))).Named <ITransform>("iif"); builder.Register((c, p) => new GeohashEncodeTransform(p.Positional <IContext>(0))).Named <ITransform>("geohashencode"); builder.Register((c, p) => new GeohashNeighborTransform(p.Positional <IContext>(0))).Named <ITransform>("geohashneighbor"); builder.Register((c, p) => new CommonPrefixTransform(p.Positional <IContext>(0))).Named <ITransform>("commonprefix"); builder.Register((c, p) => new CommonPrefixesTransform(p.Positional <IContext>(0))).Named <ITransform>("commonprefixes"); builder.Register((c, p) => new DistanceTransform(p.Positional <IContext>(0))).Named <ITransform>("distance"); builder.Register((c, p) => new FilterTransform(p.Positional <IContext>(0), FilterType.Include)).Named <ITransform>("include"); builder.Register((c, p) => new FilterTransform(p.Positional <IContext>(0), FilterType.Exclude)).Named <ITransform>("exclude"); builder.Register((c, p) => new FromSplitTransform(p.Positional <IContext>(0))).Named <ITransform>("fromsplit"); builder.Register((c, p) => new FromRegexTransform(p.Positional <IContext>(0))).Named <ITransform>("fromregex"); builder.Register((c, p) => new FromLengthsTranform(p.Positional <IContext>(0))).Named <ITransform>("fromlengths"); // return true or false, validators builder.Register((c, p) => new AnyTransform(p.Positional <IContext>(0))).Named <ITransform>("any"); builder.Register((c, p) => new StartsWithTransform(p.Positional <IContext>(0))).Named <ITransform>("startswith"); builder.Register((c, p) => new EndsWithTransform(p.Positional <IContext>(0))).Named <ITransform>("endswith"); builder.Register((c, p) => new InTransform(p.Positional <IContext>(0))).Named <ITransform>("in"); builder.Register((c, p) => new ContainsTransform(p.Positional <IContext>(0))).Named <ITransform>("contains"); builder.Register((c, p) => new IsTransform(p.Positional <IContext>(0))).Named <ITransform>("is"); builder.Register((c, p) => new EqualsTransform(p.Positional <IContext>(0))).Named <ITransform>("equal"); builder.Register((c, p) => new EqualsTransform(p.Positional <IContext>(0))).Named <ITransform>("equals"); builder.Register((c, p) => new IsEmptyTransform(p.Positional <IContext>(0))).Named <ITransform>("isempty"); builder.Register((c, p) => new IsDefaultTransform(p.Positional <IContext>(0))).Named <ITransform>("isdefault"); builder.Register((c, p) => new IsNumericTransform(p.Positional <IContext>(0))).Named <ITransform>("isnumeric"); builder.Register((c, p) => new RegexIsMatchTransform(p.Positional <IContext>(0))).Named <ITransform>("ismatch"); builder.Register((c, p) => new GeocodeTransform(p.Positional <IContext>(0))).Named <ITransform>("fromaddress"); builder.Register((c, p) => new DateMathTransform(p.Positional <IContext>(0))).Named <ITransform>("datemath"); builder.Register((c, p) => new IsDaylightSavingsTransform(p.Positional <IContext>(0))).Named <ITransform>("isdaylightsavings"); builder.Register((c, p) => new SlugifyTransform(p.Positional <IContext>(0))).Named <ITransform>("slugify"); // wip builder.Register((c, p) => new WebTransform(p.Positional <IContext>(0))).Named <ITransform>("web"); builder.Register((c, p) => new UrlEncodeTransform(p.Positional <IContext>(0))).Named <ITransform>("urlencode"); builder.Register((c, p) => new FromJsonTransform(p.Positional <IContext>(0), o => JsonConvert.SerializeObject(o, Formatting.None))).Named <ITransform>("fromjson"); builder.Register((c, p) => new DistinctTransform(p.Positional <IContext>(0))).Named <ITransform>("distinct"); builder.Register((c, p) => new RegexMatchCountTransform(p.Positional <IContext>(0))).Named <ITransform>("matchcount"); builder.Register((c, p) => new AppendTransform(p.Positional <IContext>(0))).Named <ITransform>("append"); builder.Register((c, p) => new PrependTransform(p.Positional <IContext>(0))).Named <ITransform>("prepend"); builder.Register((c, p) => new AppendTransform(p.Positional <IContext>(0))).Named <ITransform>("prefix"); builder.Register((c, p) => new PrependTransform(p.Positional <IContext>(0))).Named <ITransform>("suffix"); builder.Register((c, p) => { var context = p.Positional <IContext>(0); return(context.Operation.Mode == "all" || context.Field.Engine != "auto" ? new Transforms.Xml.FromXmlTransform(context, c.ResolveNamed <IRowFactory>(context.Entity.Key, new NamedParameter("capacity", context.GetAllEntityFields().Count()))) : new Transforms.FromXmlTransform(context) as ITransform); }).Named <ITransform>("fromxml"); builder.Register((c, p) => new JavascriptTransform(new ChakraCoreJsEngineFactory(), p.Positional <IContext>(0), c.Resolve <IReader>())).Named <ITransform>("js"); builder.Register((c, p) => new JavascriptTransform(new ChakraCoreJsEngineFactory(), p.Positional <IContext>(0), c.Resolve <IReader>())).Named <ITransform>("javascript"); builder.Register((c, p) => new JavascriptTransform(new ChakraCoreJsEngineFactory(), p.Positional <IContext>(0), c.Resolve <IReader>())).Named <ITransform>("chakra"); }
protected BaseTransform(PipelineContext context) { context.Activity = PipelineActivity.Transform; Context = context; Context.Debug("Registered"); }
public ILifetimeScope CreateScope(Process process, IPipelineLogger logger) { var builder = new ContainerBuilder(); #if PLUGINS builder.Properties["Process"] = process; #endif builder.Register(ctx => process).As <Process>(); builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance(); // register short-hand for t attribute, allowing for additional transforms var transformModule = new TransformModule(process, _methods, _shortHand, logger); foreach (var t in _transforms) { transformModule.AddTransform(t); } builder.RegisterModule(transformModule); // register short-hand for v attribute, allowing for additional validators var validateModule = new ValidateModule(process, _methods, _shortHand, logger); foreach (var v in _validators) { validateModule.AddValidator(v); } builder.RegisterModule(validateModule); builder.RegisterModule(new InternalModule(process)); #if PLUGINS // just in case other modules need to see these builder.Properties["ShortHand"] = _shortHand; builder.Properties["Methods"] = _methods; #endif // allowing for additional modules foreach (var module in _modules) { builder.RegisterModule(module); } // Process Context builder.Register <IContext>((ctx, p) => new PipelineContext(logger, process)).As <IContext>(); // Process Output Context builder.Register(ctx => { var context = ctx.Resolve <IContext>(); return(new OutputContext(context)); }).As <OutputContext>(); // Connection and Process Level Output Context foreach (var connection in process.Connections) { builder.Register(ctx => new ConnectionContext(ctx.Resolve <IContext>(), connection)).Named <IConnectionContext>(connection.Key); if (connection.Name != "output") { continue; } // register output for connection builder.Register(ctx => { var context = ctx.ResolveNamed <IConnectionContext>(connection.Key); return(new OutputContext(context)); }).Named <OutputContext>(connection.Key); } // Entity Context and RowFactory foreach (var entity in process.Entities) { builder.Register <IContext>((ctx, p) => new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity)).Named <IContext>(entity.Key); builder.Register(ctx => { var context = ctx.ResolveNamed <IContext>(entity.Key); return(new InputContext(context)); }).Named <InputContext>(entity.Key); builder.Register <IRowFactory>((ctx, p) => new RowFactory(p.Named <int>("capacity"), entity.IsMaster, false)).Named <IRowFactory>(entity.Key); builder.Register(ctx => { var context = ctx.ResolveNamed <IContext>(entity.Key); return(new OutputContext(context)); }).Named <OutputContext>(entity.Key); var connection = process.Connections.First(c => c.Name == entity.Connection); builder.Register(ctx => new ConnectionContext(ctx.Resolve <IContext>(), connection)).Named <IConnectionContext>(entity.Key); } // entity pipelines foreach (var entity in process.Entities) { builder.Register(ctx => { var type = process.Pipeline == "defer" ? entity.Pipeline : process.Pipeline; var context = ctx.ResolveNamed <IContext>(entity.Key); context.Debug(() => $"Registering {type} for entity {entity.Alias}."); var outputController = ctx.IsRegisteredWithName <IOutputController>(entity.Key) ? ctx.ResolveNamed <IOutputController>(entity.Key) : new NullOutputController(); var pipeline = new DefaultPipeline(outputController, context); // TODO: rely on IInputProvider's Read method instead (after every provider has one) pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IRead)) ? ctx.ResolveNamed <IRead>(entity.Key) : null); pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IInputProvider)) ? ctx.ResolveNamed <IInputProvider>(entity.Key) : null); // transforms if (!process.ReadOnly) { pipeline.Register(new SetSystemFields(new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity))); } pipeline.Register(new IncrementTransform(context)); pipeline.Register(new DefaultTransform(context, context.GetAllEntityFields().Where(f => !f.System))); pipeline.Register(TransformFactory.GetTransforms(ctx, context, entity.GetAllFields().Where(f => f.Transforms.Any()))); pipeline.Register(ValidateFactory.GetValidators(ctx, context, entity.GetAllFields().Where(f => f.Validators.Any()))); if (!process.ReadOnly) { pipeline.Register(new StringTruncateTransfom(new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity))); } pipeline.Register(new LogTransform(context)); // writer, TODO: rely on IOutputProvider instead pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IWrite)) ? ctx.ResolveNamed <IWrite>(entity.Key) : null); pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IOutputProvider)) ? ctx.ResolveNamed <IOutputProvider>(entity.Key) : null); // updater pipeline.Register(process.ReadOnly || !ctx.IsRegisteredWithName(entity.Key, typeof(IUpdate)) ? new NullUpdater() : ctx.ResolveNamed <IUpdate>(entity.Key)); return(pipeline); }).Named <IPipeline>(entity.Key); } // process pipeline builder.Register(ctx => { var calc = process.ToCalculatedFieldsProcess(); var entity = calc.Entities.First(); var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity); var outputContext = new OutputContext(context); context.Debug(() => $"Registering {process.Pipeline} pipeline."); var outputController = ctx.IsRegistered <IOutputController>() ? ctx.Resolve <IOutputController>() : new NullOutputController(); var pipeline = new DefaultPipeline(outputController, context); // no updater necessary pipeline.Register(new NullUpdater(context, false)); if (!process.CalculatedFields.Any()) { pipeline.Register(new NullReader(context, false)); pipeline.Register(new NullWriter(context, false)); return(pipeline); } // register transforms pipeline.Register(new IncrementTransform(context)); pipeline.Register(new LogTransform(context)); pipeline.Register(new DefaultTransform(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity), entity.CalculatedFields)); pipeline.Register(TransformFactory.GetTransforms(ctx, context, entity.CalculatedFields)); pipeline.Register(ValidateFactory.GetValidators(ctx, context, entity.GetAllFields().Where(f => f.Validators.Any()))); pipeline.Register(new StringTruncateTransfom(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity))); // register input and output pipeline.Register(ctx.IsRegistered <IRead>() ? ctx.Resolve <IRead>() : new NullReader(context)); pipeline.Register(ctx.IsRegistered <IWrite>() ? ctx.Resolve <IWrite>() : new NullWriter(context)); if (outputContext.Connection.Provider == "sqlserver") { pipeline.Register(new MinDateTransform(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity), new DateTime(1753, 1, 1))); } return(pipeline); }).As <IPipeline>(); // process controller builder.Register <IProcessController>(ctx => { var pipelines = new List <IPipeline>(); // entity-level pipelines foreach (var entity in process.Entities) { var pipeline = ctx.ResolveNamed <IPipeline>(entity.Key); pipelines.Add(pipeline); if (entity.Delete && process.Mode != "init") { pipeline.Register(ctx.ResolveNamed <IEntityDeleteHandler>(entity.Key)); } } // process-level pipeline for process level calculated fields if (ctx.IsRegistered <IPipeline>()) { pipelines.Add(ctx.Resolve <IPipeline>()); } var context = ctx.Resolve <IContext>(); var controller = new ProcessController(pipelines, context); // output initialization if (process.Mode == "init" && ctx.IsRegistered <IInitializer>()) { controller.PreActions.Add(ctx.Resolve <IInitializer>()); } // flatten(ing) is first post-action var isAdo = Constants.AdoProviderSet().Contains(process.Output().Provider); if (process.Flatten && isAdo) { if (ctx.IsRegisteredWithName <IAction>(process.Output().Key)) { controller.PostActions.Add(ctx.ResolveNamed <IAction>(process.Output().Key)); } else { context.Error($"Could not find ADO Flatten Action for provider {process.Output().Provider}."); } } // actions foreach (var action in process.Actions.Where(a => a.GetModes().Any(m => m == process.Mode || m == "*"))) { if (action.Before) { controller.PreActions.Add(ctx.ResolveNamed <IAction>(action.Key)); } if (action.After) { controller.PostActions.Add(ctx.ResolveNamed <IAction>(action.Key)); } } return(controller); }).As <IProcessController>(); var build = builder.Build(); return(build.BeginLifetimeScope()); }
protected override void RegisterProcess(ContainerBuilder builder, Process original) { // I need to create a new process with an entity with the appropriate fields // clone process, remove entities, and create entity needed for calcuted fields var calc = original.Clone() as Process; calc.Entities.Clear(); calc.CalculatedFields.Clear(); calc.Relationships.Clear(); var entity = original.GetDefaultOf <Entity>(e => { e.Name = "sys.Calc"; e.Connection = "output"; e.IsMaster = false; e.Fields = new List <Field> { original.GetDefaultOf <Field>(f => { f.Name = Constants.TflKey; f.Type = "int"; f.PrimaryKey = true; }) }; }); // Add fields that calculated fields depend on entity.Fields.AddRange(original.CalculatedFields .SelectMany(f => f.Transforms) .SelectMany(t => t.Parameters) .Where(p => !p.HasValue() && p.IsField(original)) .Select(p => p.AsField(original).Clone() as Field) ); entity.CalculatedFields.AddRange(original.CalculatedFields.Select(cf => cf.Clone() as Field)); calc.Entities.Add(entity); calc.ModifyKeys(); calc.ModifyIndexes(); // I need a process keyed pipeline builder.Register((ctx) => { var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity); IPipeline pipeline; switch (original.Pipeline) { case "parallel.linq": context.Debug("Registering {0} pipeline.", original.Pipeline); pipeline = new ParallelPipeline(new DefaultPipeline(new NullEntityController(), context)); break; default: context.Debug("Registering linq pipeline.", original.Pipeline); pipeline = new DefaultPipeline(new NullEntityController(), context); break; } // register transforms pipeline.Register(new DefaultTransform(context, entity.CalculatedFields)); pipeline.Register(TransformFactory.GetTransforms(ctx, calc, entity, entity.CalculatedFields)); pipeline.Register(new StringTruncateTransfom(context)); // register input and output var outputContext = new OutputContext(context, new Incrementer(context)); switch (outputContext.Connection.Provider) { case "sqlserver": pipeline.Register(new SqlStarParametersReader(outputContext, original)); pipeline.Register(new SqlCalculatedFieldUpdater(outputContext, original)); pipeline.Register(new MinDateTransform(context, new DateTime(1753, 1, 1))); break; default: pipeline.Register(new NullReader(context)); pipeline.Register(new NullWriter(context)); break; } // no updater necessary pipeline.Register(new NullUpdater(context, false)); return(pipeline); }).Named <IPipeline>(original.Key); }
protected override void Load(ContainerBuilder builder) { if (_process == null) { return; } var calc = _process.ToCalculatedFieldsProcess(); var entity = calc.Entities.First(); // I need a process keyed pipeline builder.Register(ctx => { var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity); var outputContext = new OutputContext(context, new Incrementer(context)); IPipeline pipeline; context.Debug(() => $"Registering {_process.Pipeline} pipeline."); var outputController = ctx.IsRegistered <IOutputController>() ? ctx.Resolve <IOutputController>() : new NullOutputController(); switch (_process.Pipeline) { case "parallel.linq": pipeline = new ParallelPipeline(new DefaultPipeline(outputController, context)); break; default: pipeline = new DefaultPipeline(outputController, context); break; } // no updater necessary pipeline.Register(new NullUpdater(context, false)); if (!_process.CalculatedFields.Any()) { pipeline.Register(new NullReader(context, false)); pipeline.Register(new NullWriter(context, false)); return(pipeline); } // register transforms pipeline.Register(new DefaultTransform(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity), entity.CalculatedFields)); pipeline.Register(TransformFactory.GetTransforms(ctx, calc, entity, entity.CalculatedFields)); pipeline.Register(new StringTruncateTransfom(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity))); // register input and output switch (outputContext.Connection.Provider) { case "sqlserver": pipeline.Register(ctx.Resolve <IRead>()); pipeline.Register(ctx.Resolve <IWrite>()); pipeline.Register(new MinDateTransform(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity), new DateTime(1753, 1, 1))); break; case "mysql": case "postgresql": case "sqlce": case "sqlite": pipeline.Register(ctx.Resolve <IRead>()); pipeline.Register(ctx.Resolve <IWrite>()); break; default: pipeline.Register(new NullReader(context)); pipeline.Register(new NullWriter(context)); break; } return(pipeline); }).As <IPipeline>(); }
protected BaseTransform(PipelineContext context) { context.Activity = PipelineActivity.Transform; Context = context; Context.Debug("Registered"); }
public static ILifetimeScope Create(Process process, IPipelineLogger logger, string placeHolderStyle) { var loadContext = new PipelineContext(logger, process); if (process.OutputIsConsole()) { logger.SuppressConsole(); } var builder = new ContainerBuilder(); builder.Properties["Process"] = process; builder.Register(ctx => placeHolderStyle).Named <string>("placeHolderStyle"); builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance(); /* this stuff is loaded (again) because tfl actions can create processes, which will need short-hand to expand configuration in advance */ builder.RegisterCallback(new TransformModule(process, logger).Configure); builder.RegisterCallback(new ShorthandTransformModule().Configure); builder.RegisterCallback(new ValidateModule().Configure); builder.RegisterCallback(new ShorthandValidateModule().Configure); builder.RegisterCallback(new RootModule().Configure); builder.RegisterCallback(new ContextModule(process).Configure); // provider loading section var providers = new HashSet <string>(process.Connections.Select(c => c.Provider).Distinct(), StringComparer.OrdinalIgnoreCase); builder.RegisterCallback(new InternalModule(process).Configure); builder.RegisterCallback(new AdoModule(process).Configure); if (providers.Contains("console")) { builder.RegisterCallback(new ConsoleModule(process).Configure); } if (providers.Contains("file")) { builder.RegisterCallback(new FileModule(process).Configure); } if (providers.Contains("geojson")) { builder.RegisterCallback(new GeoJsonModule(process).Configure); } if (providers.Contains("kml")) { builder.RegisterCallback(new KmlModule(process).Configure); } if (providers.Contains("folder")) { builder.RegisterCallback(new FolderModule(process).Configure); } if (providers.Contains("filesystem")) { builder.RegisterCallback(new FileSystemModule(process).Configure); } if (providers.Contains("excel")) { builder.RegisterCallback(new ExcelModule(process).Configure); } if (providers.Contains("web")) { builder.RegisterCallback(new WebModule(process).Configure); } if (providers.Contains("rethinkdb")) { builder.RegisterCallback(new RethinkDBModule(process).Configure); } var pluginsFolder = Path.Combine(AssemblyDirectory, "plugins"); if (Directory.Exists(pluginsFolder)) { var assemblies = new List <Assembly>(); foreach (var file in Directory.GetFiles(pluginsFolder, "Transformalize.Provider.*.Autofac.dll", SearchOption.TopDirectoryOnly)) { var info = new FileInfo(file); var name = info.Name.ToLower().Split('.').FirstOrDefault(f => f != "dll" && f != "transformalize" && f != "provider" && f != "autofac"); if (!providers.Contains(name)) { continue; } loadContext.Debug(() => $"Loading {name} provider"); var assembly = Assembly.LoadFile(new FileInfo(file).FullName); assemblies.Add(assembly); } if (assemblies.Any()) { builder.RegisterAssemblyModules(assemblies.ToArray()); } } // template providers builder.RegisterCallback(new RazorModule(process).Configure); // etc builder.RegisterCallback(new MapModule(process).Configure); builder.RegisterCallback(new TemplateModule(process).Configure); builder.RegisterCallback(new ActionModule(process).Configure); builder.RegisterCallback(new EntityPipelineModule(process).Configure); builder.RegisterCallback(new ProcessPipelineModule(process).Configure); builder.RegisterCallback(new ProcessControlModule(process).Configure); return(builder.Build().BeginLifetimeScope()); }
protected override void RegisterProcess(ContainerBuilder builder, Process original) { // I need to create a new process with an entity with the appropriate fields // clone process, remove entities, and create entity needed for calcuted fields var calc = original.Clone() as Process; calc.Entities.Clear(); calc.CalculatedFields.Clear(); calc.Relationships.Clear(); var entity = original.GetDefaultOf<Entity>(e => { e.Name = "sys.Calc"; e.Connection = "output"; e.IsMaster = false; e.Fields = new List<Field> { original.GetDefaultOf<Field>(f => { f.Name = Constants.TflKey; f.Type = "int"; f.PrimaryKey = true; }) }; }); // Add fields that calculated fields depend on entity.Fields.AddRange(original.CalculatedFields .SelectMany(f => f.Transforms) .SelectMany(t => t.Parameters) .Where(p => !p.HasValue() && p.IsField(original)) .Select(p => p.AsField(original).Clone() as Field) ); entity.CalculatedFields.AddRange(original.CalculatedFields.Select(cf => cf.Clone() as Field)); calc.Entities.Add(entity); calc.ModifyKeys(); calc.ModifyIndexes(); // I need a process keyed pipeline builder.Register((ctx) => { var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), calc, entity); IPipeline pipeline; switch (original.Pipeline) { case "parallel.linq": context.Debug("Registering {0} pipeline.", original.Pipeline); pipeline = new ParallelPipeline(new DefaultPipeline(new NullEntityController(), context)); break; default: context.Debug("Registering linq pipeline.", original.Pipeline); pipeline = new DefaultPipeline(new NullEntityController(), context); break; } // register transforms pipeline.Register(new DefaultTransform(context, entity.CalculatedFields)); pipeline.Register(TransformFactory.GetTransforms(ctx, calc, entity, entity.CalculatedFields)); pipeline.Register(new StringTruncateTransfom(context)); // register input and output var outputContext = new OutputContext(context, new Incrementer(context)); switch (outputContext.Connection.Provider) { case "sqlserver": pipeline.Register(new SqlStarParametersReader(outputContext, original)); pipeline.Register(new SqlCalculatedFieldUpdater(outputContext, original)); pipeline.Register(new MinDateTransform(context, new DateTime(1753, 1, 1))); break; default: pipeline.Register(new NullReader(context)); pipeline.Register(new NullWriter(context)); break; } // no updater necessary pipeline.Register(new NullUpdater(context, false)); return pipeline; }).Named<IPipeline>(original.Key); }
public ILifetimeScope CreateScope(ILifetimeScope scope, IPipelineLogger logger) { var process = scope.Resolve <Process>(); var builder = new ContainerBuilder(); builder.Properties["Process"] = process; builder.Register((ctx) => process).As <Process>(); builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance(); builder.RegisterModule(new CsScriptModule()); builder.RegisterModule(new BogusModule()); RegisterTransform(builder, c => new HashcodeTransform(c), new[] { new OperationSignature("hashcode") }); // Process Context builder.Register <IContext>((ctx, p) => new PipelineContext(logger, process)).As <IContext>(); // Process Output Context builder.Register(ctx => { var context = ctx.Resolve <IContext>(); return(new OutputContext(context)); }).As <OutputContext>(); // Connection and Process Level Output Context foreach (var connection in process.Connections) { builder.Register(ctx => new ConnectionContext(ctx.Resolve <IContext>(), connection)).Named <IConnectionContext>(connection.Key); if (connection.Name != "output") { continue; } // register output for connection builder.Register(ctx => { var context = ctx.ResolveNamed <IConnectionContext>(connection.Key); return(new OutputContext(context)); }).Named <OutputContext>(connection.Key); } // Entity Context and RowFactory foreach (var entity in process.Entities) { builder.Register <IContext>((ctx, p) => new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity)).Named <IContext>(entity.Key); builder.Register(ctx => { var context = ctx.ResolveNamed <IContext>(entity.Key); return(new InputContext(context)); }).Named <InputContext>(entity.Key); builder.Register <IRowFactory>((ctx, p) => new RowFactory(p.Named <int>("capacity"), entity.IsMaster, false)).Named <IRowFactory>(entity.Key); builder.Register(ctx => { var context = ctx.ResolveNamed <IContext>(entity.Key); return(new OutputContext(context)); }).Named <OutputContext>(entity.Key); var connection = process.Connections.First(c => c.Name == entity.Connection); builder.Register(ctx => new ConnectionContext(ctx.Resolve <IContext>(), connection)).Named <IConnectionContext>(entity.Key); } // Entity input foreach (var entity in process.Entities.Where(e => process.Connections.First(c => c.Name == e.Connection).Provider == "internal")) { builder.RegisterType <NullInputProvider>().Named <IInputProvider>(entity.Key); // READER builder.Register <IRead>(ctx => { var input = ctx.ResolveNamed <InputContext>(entity.Key); var rowFactory = ctx.ResolveNamed <IRowFactory>(entity.Key, new NamedParameter("capacity", input.RowCapacity)); return(new InternalReader(input, rowFactory)); }).Named <IRead>(entity.Key); } // PROCESS OUTPUT CONTROLLER builder.Register <IOutputController>(ctx => new NullOutputController()).As <IOutputController>(); // Entity Output foreach (var entity in process.Entities) { builder.Register <IOutputController>(ctx => new NullOutputController()).Named <IOutputController>(entity.Key); builder.Register <IOutputProvider>(ctx => new InternalOutputProvider(ctx.ResolveNamed <OutputContext>(entity.Key), ctx.ResolveNamed <IWrite>(entity.Key))).Named <IOutputProvider>(entity.Key); // WRITER builder.Register <IWrite>(ctx => new InternalWriter(ctx.ResolveNamed <OutputContext>(entity.Key))).Named <IWrite>(entity.Key); } // entity pipelines foreach (var entity in process.Entities) { builder.Register(ctx => { var context = ctx.ResolveNamed <IContext>(entity.Key); var outputController = ctx.IsRegisteredWithName <IOutputController>(entity.Key) ? ctx.ResolveNamed <IOutputController>(entity.Key) : new NullOutputController(); IPipeline pipeline = new DefaultPipeline(outputController, context); // TODO: rely on IInputProvider's Read method instead (after every provider has one) pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IRead)) ? ctx.ResolveNamed <IRead>(entity.Key) : null); pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IInputProvider)) ? ctx.ResolveNamed <IInputProvider>(entity.Key) : null); // transforms if (!process.ReadOnly) { pipeline.Register(new SetSystemFields(new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity))); } pipeline.Register(new IncrementTransform(context)); pipeline.Register(new LogTransform(context)); pipeline.Register(new DefaultTransform(context, context.GetAllEntityFields().Where(f => !f.System))); pipeline.Register(TransformFactory.GetTransforms(ctx, context, entity.GetAllFields().Where(f => f.Transforms.Any()))); if (!process.ReadOnly) { pipeline.Register(new StringTruncateTransfom(new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity))); } pipeline.Register(new LogTransform(context)); // writer, TODO: rely on IOutputProvider instead pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IWrite)) ? ctx.ResolveNamed <IWrite>(entity.Key) : null); pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IOutputProvider)) ? ctx.ResolveNamed <IOutputProvider>(entity.Key) : null); // updater pipeline.Register(process.ReadOnly || !ctx.IsRegisteredWithName(entity.Key, typeof(IUpdate)) ? new NullUpdater() : ctx.ResolveNamed <IUpdate>(entity.Key)); return(pipeline); }).Named <IPipeline>(entity.Key); } // process pipeline builder.Register(ctx => { var calc = process.ToCalculatedFieldsProcess(); var entity = calc.Entities.First(); var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity); context.Debug(() => $"Registering {process.Pipeline} pipeline."); var outputController = ctx.IsRegistered <IOutputController>() ? ctx.Resolve <IOutputController>() : new NullOutputController(); IPipeline pipeline = new DefaultPipeline(outputController, context); // no updater necessary pipeline.Register(new NullUpdater(context, false)); if (!process.CalculatedFields.Any()) { pipeline.Register(new NullReader(context, false)); pipeline.Register(new NullWriter(context, false)); return(pipeline); } // register transforms pipeline.Register(new IncrementTransform(context)); pipeline.Register(new LogTransform(context)); pipeline.Register(new DefaultTransform(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity), entity.CalculatedFields)); pipeline.Register(TransformFactory.GetTransforms(ctx, context, entity.CalculatedFields)); pipeline.Register(new StringTruncateTransfom(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity))); pipeline.Register(new NullReader(context)); pipeline.Register(new NullWriter(context)); return(pipeline); }).As <IPipeline>(); // process controller builder.Register <IProcessController>(ctx => { var pipelines = new List <IPipeline>(); // entity-level pipelines foreach (var entity in process.Entities) { var pipeline = ctx.ResolveNamed <IPipeline>(entity.Key); pipelines.Add(pipeline); if (entity.Delete && process.Mode != "init") { pipeline.Register(ctx.ResolveNamed <IEntityDeleteHandler>(entity.Key)); } } // process-level pipeline for process level calculated fields if (ctx.IsRegistered <IPipeline>()) { pipelines.Add(ctx.Resolve <IPipeline>()); } var context = ctx.Resolve <IContext>(); var controller = new ProcessController(pipelines, context); // actions foreach (var action in process.Actions.Where(a => a.GetModes().Any(m => m == process.Mode || m == "*"))) { if (action.Before) { controller.PreActions.Add(ctx.ResolveNamed <IAction>(action.Key)); } if (action.After) { controller.PostActions.Add(ctx.ResolveNamed <IAction>(action.Key)); } } return(controller); }).As <IProcessController>(); var build = builder.Build(); return(build.BeginLifetimeScope()); }
protected override void Load(ContainerBuilder builder) { var loadContext = new PipelineContext(_logger, _process); // set properties for plugins builder.Properties["ShortHand"] = _shortHand; builder.Properties["Methods"] = _methods; builder.Properties["Process"] = _process; // new method so transform author can define shorthand signature(s) RegisterTransform(builder, (ctx, c) => new AbsTransform(c), new AbsTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new SliceTransform(c), new SliceTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new FormatTransform(c), new FormatTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new AddTransform(c), new AddTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new EqualsTransform(c), new EqualsTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new CeilingTransform(c), new CeilingTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new CoalesceTransform(c), new CoalesceTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new ConcatTransform(c), new ConcatTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new ConnectionTransform(c), new ConnectionTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new ConvertTransform(c), new ConvertTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new CopyTransform(c), new CopyTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new DateDiffTransform(c), new DateDiffTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new DatePartTransform(c), new DatePartTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new DecompressTransform(c), new DecompressTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new CompressTransform(c), new CompressTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new FileExtTransform(c), new FileExtTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new FileNameTransform(c), new FileNameTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new FilePathTransform(c), new FilePathTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new SplitTransform(c), new SplitTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new LastDayTransform(c), new LastDayTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new LastTransform(c), new LastTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new FirstTransform(c), new FirstTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new FloorTransform(c), new FloorTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new FormatXmlTransform(c), new FormatXmlTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new FormatPhoneTransform(c), new FormatPhoneTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new HashcodeTransform(c), new HashcodeTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new DecodeTransform(c), new DecodeTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new HtmlEncodeTransform(c), new HtmlEncodeTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new InsertTransform(c), new InsertTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new InvertTransform(c), new InvertTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new JoinTransform(c), new JoinTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new LeftTransform(c), new LeftTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new RightTransform(c), new RightTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new ToLowerTransform(c), new ToLowerTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new MapTransform(c), new MapTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new RegexMatchTransform(c), new RegexMatchTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new RegexMatchingTransform(c), new RegexMatchingTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new MultiplyTransform(c), new MultiplyTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new NextDayTransform(c), new NextDayTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new UtcNowTransform(c), new UtcNowTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new RowNumberTransform(c), new RowNumberTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new TimeZoneTransform(c), new TimeZoneTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new TimeZoneOffsetTransform(c), new TimeZoneOffsetTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new OppositeTransform(c), new OppositeTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new DateAddTransform(c), new DateAddTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new TimeAheadTransform(c), new TimeAheadTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new TimeAgoTransform(c), new TimeAgoTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new PadLeftTransform(c), new PadLeftTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new PadRightTransform(c), new PadRightTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new RegexReplaceTransform(c), new RegexReplaceTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new RemoveTransform(c), new RemoveTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new ReplaceTransform(c), new ReplaceTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new RoundTransform(c), new RoundTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new RoundToTransform(c), new RoundToTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new RoundUpToTransform(c), new RoundUpToTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new RoundDownToTransform(c), new RoundDownToTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new SplitLengthTransform(c), new SplitLengthTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new SubStringTransform(c), new SubStringTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new TagTransform(c), new TagTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new ToStringTransform(c), new ToStringTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new ToTimeTransform(c), new ToTimeTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new ToYesNoTransform(c), new ToYesNoTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new TrimTransform(c), new TrimTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new TrimStartTransform(c), new TrimStartTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new TrimEndTransform(c), new TrimEndTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new LineTransform(c), new LineTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new VelocityTransform(c, ctx.Resolve <IReader>()), new VelocityTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new ToUpperTransform(c), new ToUpperTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new XPathTransform(c), new XPathTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new IIfTransform(c), new IIfTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new AppendTransform(c), new AppendTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new PrependTransform(c), new PrependTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new GeohashEncodeTransform(c), new GeohashEncodeTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new GeohashNeighborTransform(c), new GeohashNeighborTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new CommonPrefixTransform(c), new CommonPrefixTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new CommonPrefixesTransform(c), new CommonPrefixesTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new DistanceTransform(c), new DistanceTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new SlugifyTransform(c), new SlugifyTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new DateMathTransform(c), new DateMathTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new WebTransform(c), new WebTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new UrlEncodeTransform(c), new UrlEncodeTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new DistinctTransform(c), new DistinctTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new RegexMatchCountTransform(c), new RegexMatchCountTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new CondenseTransform(c), new CondenseTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new RandomTransform(c), new RandomTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new IdentityTransform(c), new IdentityTransform().GetSignatures()); // row filtering RegisterTransform(builder, (ctx, c) => new FilterTransform(FilterType.Include, c), new FilterTransform(FilterType.Include).GetSignatures()); RegisterTransform(builder, (ctx, c) => new FilterTransform(FilterType.Exclude, c), new FilterTransform(FilterType.Exclude).GetSignatures()); // field producing RegisterTransform(builder, (ctx, c) => new FromSplitTransform(c), new FromSplitTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new FromRegexTransform(c), new FromRegexTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new FromLengthsTransform(c), new FromLengthsTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new FromJsonTransform(c, o => JsonConvert.SerializeObject(o, Formatting.None)), new FromJsonTransform().GetSignatures()); // return true or false transforms RegisterTransform(builder, (ctx, c) => new AnyTransform(c), new AnyTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new StartsWithTransform(c), new StartsWithTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new EndsWithTransform(c), new EndsWithTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new InTransform(c), new InTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new ContainsTransform(c), new ContainsTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new IsTransform(c), new IsTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new EqualsTransform(c), new EqualsTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new IsEmptyTransform(c), new IsEmptyTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new IsDefaultTransform(c), new IsDefaultTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new IsNumericTransform(c), new IsNumericTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new RegexIsMatchTransform(c), new RegexIsMatchTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new IsDaylightSavingsTransform(c), new IsDaylightSavingsTransform().GetSignatures()); RegisterTransform(builder, (ctx, c) => new ToBoolTransform(c), new ToBoolTransform().GetSignatures()); // uncategorized RegisterTransform(builder, (ctx, c) => new LogTransform(c), new LogTransform().GetSignatures()); // xml RegisterTransform(builder, (ctx, c) => c.Operation.Mode == "all" || c.Field.Engine != "auto" ? new Transforms.Xml.FromXmlTransform(ctx.ResolveNamed <IRowFactory>(c.Entity.Key, new NamedParameter("capacity", c.GetAllEntityFields().Count())), c) : new Transforms.FromXmlTransform(c) as ITransform, new[] { new OperationSignature("fromxml") } ); RegisterTransform(builder, (ctx, c) => new JavascriptTransform(new ChakraCoreJsEngineFactory(), ctx.Resolve <IReader>(), c), new JavascriptTransform(null, null).GetSignatures()); var pluginsFolder = Path.Combine(AssemblyDirectory, "plugins"); if (Directory.Exists(pluginsFolder)) { var assemblies = new List <Assembly>(); foreach (var file in Directory.GetFiles(pluginsFolder, "Transformalize.Transform.*.Autofac.dll", SearchOption.TopDirectoryOnly)) { var info = new FileInfo(file); var name = info.Name.ToLower().Split('.').FirstOrDefault(f => f != "dll" && f != "transformalize" && f != "transform" && f != "autofac"); loadContext.Debug(() => $"Loading {name} transform(s)"); var assembly = Assembly.LoadFile(new FileInfo(file).FullName); assemblies.Add(assembly); } if (assemblies.Any()) { builder.RegisterAssemblyModules(assemblies.ToArray()); } } // register the short hand builder.Register((c, p) => _shortHand).Named <ShorthandRoot>(FieldsName).InstancePerLifetimeScope(); builder.Register((c, p) => _shortHand).Named <ShorthandRoot>(ParametersName).InstancePerLifetimeScope(); builder.Register((c, p) => new ShorthandCustomizer(c.ResolveNamed <ShorthandRoot>(FieldsName), new[] { "fields", "calculated-fields" }, "t", "transforms", "method")).Named <IDependency>(FieldsName).InstancePerLifetimeScope(); builder.Register((c, p) => new ShorthandCustomizer(c.ResolveNamed <ShorthandRoot>(ParametersName), new[] { "parameters" }, "t", "transforms", "method")).Named <IDependency>(ParametersName).InstancePerLifetimeScope(); }
public ILifetimeScope CreateScope(Process process, IPipelineLogger logger) { var builder = new ContainerBuilder(); // the modules below rely on this process being there builder.Properties["Process"] = process; builder.Register(ctx => process).As <Process>(); builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance(); // register short-hand for t attribute, allowing for additional transforms var tm = new TransformModule(process, _methods, _shortHand, logger) { Plugins = false }; // adding additional transforms here tm.AddTransform(new TransformHolder((c) => new UsernameTransform(_httpContext, c), new UsernameTransform().GetSignatures())); tm.AddTransform(new TransformHolder((c) => new UserIdTransform(_httpContext, _userService, c), new UserIdTransform().GetSignatures())); tm.AddTransform(new TransformHolder((c) => new UserEmailTransform(_httpContext, _userService, c), new UserEmailTransform().GetSignatures())); tm.AddTransform(new TransformHolder((c) => new OrchardRazorTransform(c, _memoryCache, _signal), new OrchardRazorTransform().GetSignatures())); tm.AddTransform(new TransformHolder((c) => new OrchardFluidTransform(c, _memoryCache, _signal), new OrchardFluidTransform().GetSignatures())); tm.AddTransform(new TransformHolder((c) => new OrchardJintTransform(c, new DefaultReader(new FileReader(), new WebReader()), _memoryCache, _signal), new OrchardJintTransform().GetSignatures())); tm.AddTransform(new TransformHolder((c) => new ToLocalTimeTransform(c, _clock, _localClock), new ToLocalTimeTransform().GetSignatures())); tm.AddTransform(new TransformHolder((c) => new GetEncodedUrlTransform(_httpContext, c), new GetEncodedUrlTransform().GetSignatures())); tm.AddTransform(new TransformHolder((c) => new GetDisplayUrlTransform(_httpContext, c), new GetDisplayUrlTransform().GetSignatures())); tm.AddTransform(new TransformHolder((c) => new OrchardTimeZoneTransform(c), new OrchardTimeZoneTransform().GetSignatures())); tm.AddTransform(new TransformHolder((c) => new GeocodeTransform(c), new GeocodeTransform().GetSignatures())); tm.AddTransform(new TransformHolder((c) => new PlaceTransform(c), new PlaceTransform().GetSignatures())); tm.AddTransform(new TransformHolder((c) => new FilePartTransform(c, _fileService), new FilePartTransform().GetSignatures())); builder.RegisterModule(tm); // register short-hand for v attribute, allowing for additional validators var vm = new ValidateModule(process, _methods, _shortHand, logger) { Plugins = false }; // adding additional validators here builder.RegisterModule(vm); // using custom internal module that does not handle the nested transformalize actions builder.RegisterModule(new OrchardInternalModule(process)); // handling nested transformalize actions here instead foreach (var action in process.Actions.Where(a => a.GetModes().Any(m => m == process.Mode || m == "*"))) { if (action.Type == "tfl") { builder.Register <IAction>(ctx => { return(new PipelineAction(action, _serviceProvider)); }).Named <IAction>(action.Key); } } // register providers var providers = new HashSet <string>(process.Connections.Select(c => c.Provider)); // relational databases builder.RegisterModule(new AdoProviderModule()); if (providers.Contains("sqlserver")) { builder.RegisterModule(new SqlServerModule() { ConnectionFactory = (c) => new ProfiledConnectionFactory(new SqlServerConnectionFactory(c)) }); } if (providers.Contains("postgresql")) { builder.RegisterModule(new PostgreSqlModule() { ConnectionFactory = (c) => new ProfiledConnectionFactory(new PostgreSqlConnectionFactory(c)) }); } if (providers.Contains("sqlite")) { builder.RegisterModule(new SqliteModule() { ConnectionFactory = (c) => new ProfiledConnectionFactory(new SqliteConnectionFactory(c)) }); } if (providers.Contains("mysql")) { builder.RegisterModule(new MySqlModule() { ConnectionFactory = (c) => new ProfiledConnectionFactory(new MySqlConnectionFactory(c)) }); } // search engines if (providers.Contains("elasticsearch")) { builder.RegisterModule(new ElasticsearchModule()); } // solr // lucene // importing, exporting var stream = _httpContext.HttpContext.Response.Body; if (providers.Contains("file")) { builder.RegisterModule(new CsvHelperProviderModule(stream)); } if (providers.Contains("excel")) { builder.RegisterModule(new OrchardExcelModule()); } // exporting if (providers.Contains("json")) { builder.RegisterModule(new JsonProviderModule(stream) { UseAsyncMethods = true }); } if (providers.Contains("geojson")) { builder.RegisterModule(new GeoJsonProviderModule(stream) { UseAsyncMethods = true }); } // misc if (providers.Contains("bogus")) { builder.RegisterModule(new BogusModule()); } if (providers.Contains("log") || process.Actions.Any(a => a.Type == "log")) { builder.RegisterModule(new OrchardLogModule(process)); } if (providers.Contains("mail")) { builder.RegisterModule(new MailModule()); } if (providers.Contains("aws")) { var services = new HashSet <string>(process.Connections.Where(c => c.Provider == "aws").Select(c => c.Service)); if (services.Contains("logs")) { builder.RegisterModule(new AwsCloudWatchProviderModule()); } if (services.Contains("connect")) { builder.RegisterModule(new AmazonConnectProviderModule()); } } // transform and validation modules need these properties builder.Properties["ShortHand"] = _shortHand; builder.Properties["Methods"] = _methods; // register transform modules here builder.RegisterModule(new JsonTransformModule()); builder.RegisterModule(new HumanizeModule()); builder.RegisterModule(new FileModule()); builder.RegisterModule(new AdoTransformModule()); builder.RegisterModule(new LambdaParserModule()); builder.RegisterModule(new AwsTransformModule()); // register validator modules here builder.RegisterModule(new JintValidateModule()); // process context builder.Register <IContext>((ctx, p) => new PipelineContext(logger, process)).As <IContext>(); // process output context builder.Register(ctx => { var context = ctx.Resolve <IContext>(); return(new OutputContext(context)); }).As <OutputContext>(); // connection and process level output context var formWriter = false; foreach (var connection in process.Connections) { builder.Register(ctx => new ConnectionContext(ctx.Resolve <IContext>(), connection)).Named <IConnectionContext>(connection.Key); // there can only be one form writer if (!formWriter && connection.Table != "[default]" && connection.Provider != "[default]") { builder.Register(ctx => { var context = ctx.ResolveNamed <IConnectionContext>(connection.Key); var connectionFactory = ctx.ResolveNamed <IConnectionFactory>(connection.Key); return(new AdoFormCommandWriter(context, connectionFactory)); }).As <AdoFormCommandWriter>(); formWriter = true; } if (connection.Name != process.Output) { continue; } // register output for connection builder.Register(ctx => { var context = ctx.ResolveNamed <IConnectionContext>(connection.Key); return(new OutputContext(context)); }).Named <OutputContext>(connection.Key); } // entity context and rowFactory foreach (var entity in process.Entities) { builder.Register <IContext>((ctx, p) => new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity)).Named <IContext>(entity.Key); builder.Register(ctx => { var context = ctx.ResolveNamed <IContext>(entity.Key); return(new InputContext(context)); }).Named <InputContext>(entity.Key); builder.Register <IRowFactory>((ctx, p) => new RowFactory(p.Named <int>("capacity"), entity.IsMaster, false)).Named <IRowFactory>(entity.Key); builder.Register(ctx => { var context = ctx.ResolveNamed <IContext>(entity.Key); return(new OutputContext(context)); }).Named <OutputContext>(entity.Key); var connection = process.Connections.First(c => c.Name == entity.Input); builder.Register(ctx => new ConnectionContext(ctx.Resolve <IContext>(), connection)).Named <IConnectionContext>(entity.Key); } // entity pipelines foreach (var entity in process.Entities) { builder.Register(ctx => { var output = process.GetOutputConnection(); var context = ctx.ResolveNamed <IContext>(entity.Key); var outputController = ctx.IsRegisteredWithName <IOutputController>(entity.Key) ? ctx.ResolveNamed <IOutputController>(entity.Key) : new NullOutputController(); var pipeline = new DefaultPipeline(outputController, context); if (entity.IsMaster) { if (GetReaderAlternate != null) { // Using an alternate reader (e.g. a ParameterRowReader that reads parameters into a row) var input = ctx.ResolveNamed <InputContext>(entity.Key); var rowFactory = ctx.ResolveNamed <IRowFactory>(entity.Key, new NamedParameter("capacity", input.RowCapacity)); pipeline.Register(GetReaderAlternate(input, rowFactory)); } else if (GetReaderDecorator != null && ctx.IsRegisteredWithName(entity.Key, typeof(IRead))) { // Decorating the normally registered reader (e.g. ParameterRowReader updating a row from an AdoReader) // TODO: Support IInputProvider instead of just IRead var input = ctx.ResolveNamed <InputContext>(entity.Key); var rowFactory = ctx.ResolveNamed <IRowFactory>(entity.Key, new NamedParameter("capacity", input.RowCapacity)); pipeline.Register(GetReaderDecorator(ctx.ResolveNamed <IRead>(entity.Key), input, rowFactory)); } else { pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IRead)) ? ctx.ResolveNamed <IRead>(entity.Key) : null); pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IInputProvider)) ? ctx.ResolveNamed <IInputProvider>(entity.Key) : null); } } else { // TODO: rely on IInputProvider's Read method instead (after every provider has one) pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IRead)) ? ctx.ResolveNamed <IRead>(entity.Key) : null); pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IInputProvider)) ? ctx.ResolveNamed <IInputProvider>(entity.Key) : null); } // transforms if (!process.ReadOnly) { pipeline.Register(new SetSystemFields(new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity))); } pipeline.Register(new IncrementTransform(context)); pipeline.Register(new DefaultTransform(context, context.GetAllEntityFields().Where(f => !f.System))); pipeline.Register(TransformFactory.GetTransforms(ctx, context, entity.GetAllFields().Where(f => f.Transforms.Any()))); pipeline.Register(ValidateFactory.GetValidators(ctx, context, entity.GetAllFields().Where(f => f.Validators.Any()))); if (!process.ReadOnly && !output.Provider.In("internal", "log")) { pipeline.Register(new StringTruncateTransfom(new PipelineContext(ctx.Resolve <IPipelineLogger>(), process, entity))); } pipeline.Register(new LogTransform(context)); // writer, TODO: rely on IOutputProvider instead pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IWrite)) ? ctx.ResolveNamed <IWrite>(entity.Key) : null); pipeline.Register(ctx.IsRegisteredWithName(entity.Key, typeof(IOutputProvider)) ? ctx.ResolveNamed <IOutputProvider>(entity.Key) : null); // updater pipeline.Register(process.ReadOnly || !ctx.IsRegisteredWithName(entity.Key, typeof(IUpdate)) ? new NullUpdater() : ctx.ResolveNamed <IUpdate>(entity.Key)); return(pipeline); }).Named <IPipeline>(entity.Key); } if (process.Entities.Count > 1 && process.Relationships.Any()) { // process pipeline builder.Register(ctx => { var calc = process.ToCalculatedFieldsProcess(); var entity = calc.Entities.First(); var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity); var outputContext = new OutputContext(context); context.Debug(() => $"Registering {process.Pipeline} pipeline."); var outputController = ctx.IsRegistered <IOutputController>() ? ctx.Resolve <IOutputController>() : new NullOutputController(); var pipeline = new DefaultPipeline(outputController, context); // no updater necessary pipeline.Register(new NullUpdater(context, false)); if (!process.CalculatedFields.Any()) { pipeline.Register(new NullReader(context, false)); pipeline.Register(new NullWriter(context, false)); return(pipeline); } // register transforms pipeline.Register(new IncrementTransform(context)); pipeline.Register(new LogTransform(context)); pipeline.Register(new DefaultTransform(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity), entity.CalculatedFields)); pipeline.Register(TransformFactory.GetTransforms(ctx, context, entity.CalculatedFields)); pipeline.Register(ValidateFactory.GetValidators(ctx, context, entity.GetAllFields().Where(f => f.Validators.Any()))); pipeline.Register(new StringTruncateTransfom(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity))); // register input and output pipeline.Register(ctx.IsRegistered <IRead>() ? ctx.Resolve <IRead>() : new NullReader(context)); pipeline.Register(ctx.IsRegistered <IWrite>() ? ctx.Resolve <IWrite>() : new NullWriter(context)); if (outputContext.Connection.Provider == "sqlserver") { pipeline.Register(new MinDateTransform(new PipelineContext(ctx.Resolve <IPipelineLogger>(), calc, entity), new DateTime(1753, 1, 1))); } return(pipeline); }).As <IPipeline>(); } // process controller builder.Register <IProcessController>(ctx => { var pipelines = new List <IPipeline>(); // entity-level pipelines foreach (var entity in process.Entities) { var pipeline = ctx.ResolveNamed <IPipeline>(entity.Key); pipelines.Add(pipeline); if (entity.Delete && process.Mode != "init") { pipeline.Register(ctx.ResolveNamed <IEntityDeleteHandler>(entity.Key)); } } // process-level pipeline for process level calculated fields if (ctx.IsRegistered <IPipeline>()) { pipelines.Add(ctx.Resolve <IPipeline>()); } var context = ctx.Resolve <IContext>(); var controller = new ProcessController(pipelines, context); // output initialization if (process.Mode == "init" && ctx.IsRegistered <IInitializer>()) { controller.PreActions.Add(ctx.Resolve <IInitializer>()); } // flatten(ing) is first post-action var output = process.GetOutputConnection(); var isAdo = _adoProviders.Contains(output.Provider); if (process.Flatten && isAdo) { if (ctx.IsRegisteredWithName <IAction>(output.Key)) { controller.PostActions.Add(ctx.ResolveNamed <IAction>(output.Key)); } else { context.Error($"Could not find ADO Flatten Action for provider {output.Provider}."); } } // actions foreach (var action in process.Actions.Where(a => a.GetModes().Any(m => m == process.Mode || m == "*"))) { if (ctx.IsRegisteredWithName <IAction>(action.Key)) { if (action.Before) { controller.PreActions.Add(ctx.ResolveNamed <IAction>(action.Key)); } if (action.After) { controller.PostActions.Add(ctx.ResolveNamed <IAction>(action.Key)); } } else { if (action.Type != "internal") { _logger.Warn(() => $"The action {action.Name} with type {action.Type} isn't registered."); } } } foreach (var map in process.Maps.Where(m => !string.IsNullOrEmpty(m.Query))) { controller.PreActions.Add(new MapReaderAction(context, map, ctx.ResolveNamed <IMapReader>(map.Name))); } return(controller); }).As <IProcessController>(); var build = builder.Build(); return(build.BeginLifetimeScope()); }