public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) { if (entity.Delete) { builder.Register<IEntityDeleteHandler>(ctx => { var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity); var inputConnection = process.Connections.First(c => c.Name == entity.Connection); IRead input = new NullReader(context); switch (inputConnection.Provider) { case "sqlserver": input = new SqlReader(context, entity.GetPrimaryKey(), ReadFrom.Input); break; } IRead output = new NullReader(context); IDelete deleter = new NullDeleter(context); var outputConnection = process.Connections.First(c => c.Name == "output"); switch (outputConnection.Provider) { case "sqlserver": output = new SqlReader(context, entity.GetPrimaryKey(), ReadFrom.Output); deleter = new SqlDeleter(new OutputContext(context, new Incrementer(context))); break; } return new ParallelDeleteHandler(new DefaultDeleteHandler(entity, input, output, deleter)); }).Named<IEntityDeleteHandler>(entity.Key); } }
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) { builder.Register<IRead>(ctx => { var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity); var input = new InputContext(context, new Incrementer(context)); switch (input.Connection.Provider) { case "internal": context.Debug("Registering {0} provider", input.Connection.Provider); return new DataSetEntityReader(input); case "sqlserver": if (input.Entity.ReadSize == 0) { context.Debug("Registering {0} reader", input.Connection.Provider); return new SqlInputReader(input, input.InputFields); } context.Debug("Registering {0} batch reader", input.Connection.Provider); return new SqlInputBatchReader( input, new SqlInputReader(input, input.Entity.GetPrimaryKey()) ); default: context.Warn("Registering null reader", input.Connection.Provider); return new NullEntityReader(); } }).Named<IRead>(entity.Key); }
protected override void RegisterProcess(ContainerBuilder builder, Process original) { builder.Register<IProcessController>(ctx => { var pipelines = new List<IPipeline>(); var deleteHandlers = new List<IEntityDeleteHandler>(); // entity-level pipelines foreach (var entity in original.Entities) { pipelines.Add(ctx.ResolveNamed<IPipeline>(entity.Key)); if (entity.Delete) { deleteHandlers.Add(ctx.ResolveNamed<IEntityDeleteHandler>(entity.Key)); } } // process-level pipeline pipelines.Add(ctx.ResolveNamed<IPipeline>(original.Key)); var outputProvider = original.Connections.First(c => c.Name == "output").Provider; var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), original); var controller = new ProcessController(pipelines, deleteHandlers); if (original.Mode == "init") { switch (outputProvider) { case "sqlserver": var output = new OutputContext(context, new Incrementer(context)); controller.PreActions.Add(new SqlInitializer(output)); controller.PostActions.Add(new SqlStarViewCreator(output)); break; } } // templates foreach (var template in original.Templates.Where(t => t.Enabled)) { controller.PreActions.Add(new RenderTemplateAction(template, ctx.ResolveNamed<ITemplateEngine>(template.Key))); foreach (var action in template.Actions.Where(a => a.GetModes().Any(m => m == original.Mode))) { if (action.Before) { controller.PreActions.Add(ctx.ResolveNamed<IAction>(action.Key)); } if (action.After) { controller.PostActions.Add(ctx.ResolveNamed<IAction>(action.Key)); } } } // actions foreach (var action in original.Actions.Where(a => a.GetModes().Any(m => m == original.Mode))) { if (action.Before) { controller.PreActions.Add(ctx.ResolveNamed<IAction>(action.Key)); } if (action.After) { controller.PostActions.Add(ctx.ResolveNamed<IAction>(action.Key)); } } return controller; }).Named<IProcessController>(original.Key); }
public static string SqlUpdateCalculatedFields(this OutputContext c, Process original) { var master = original.Entities.First(e => e.IsMaster); var fields = c.Entity.CalculatedFields.Where(f => f.Output && f.Name != Constants.TflKey && f.Name != Constants.TflHashCode).ToArray(); var sets = string.Join(",", fields.Select(f => "[" + original.CalculatedFields.First(cf=>cf.Name == f.Name).FieldName() + "] = @" + f.FieldName())); var sql = $"UPDATE [{master.OutputTableName(original.Name)}] SET {sets} WHERE TflKey = @TflKey;"; c.Debug(sql); return sql; }
private static IAction SwitchAction(IComponentContext ctx, Process process, Action action) { var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process); switch (action.Name) { case "copy": return action.InTemplate ? (IAction) new ContentToFileAction(context, action) : new FileToFileAction(context, action); case "web": return new WebAction(context, action); default: context.Error("{0} action is not registered.", action.Name); return new NullAction(); } }
public static IEnumerable<ITransform> GetTransforms(IComponentContext ctx, Process process, Entity entity, IEnumerable<Field> fields) { var transforms = new List<ITransform>(); foreach (var f in fields.Where(f => f.Transforms.Any())) { var field = f; if (field.RequiresCompositeValidator()) { transforms.Add(new CompositeValidator( new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity, field), field.Transforms.Select(t => SwitchTransform(ctx, new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity, field, t) { Activity = PipelineActivity.Transform })) )); } else { transforms.AddRange(field.Transforms.Select(t => SwitchTransform(ctx, new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity, field, t) { Activity = PipelineActivity.Transform }))); } } return transforms; }
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) { //master updater builder.Register<IUpdate>((ctx) => { var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity); var incrementer = new Incrementer(context); var output = new OutputContext(context, incrementer); switch (output.Connection.Provider) { case "sqlserver": context.Debug("Registering {0} master updater", output.Connection.Provider); return new SqlMasterUpdater(output); default: context.Warn("Registering null updater"); return new NullMasterUpdater(); } }).Named<IUpdate>(entity.Key); }
public Field AsField(Process process) { if (_loadedField != null) return _loadedField; if (string.IsNullOrEmpty(Entity)) { _loadedField = process.GetAllFields().FirstOrDefault(f => f.Alias == Field) ?? process.GetAllFields().FirstOrDefault(f => f.Name == Field); return _loadedField; } Entity entity; if (process.TryGetEntity(Entity, out entity)) { if (entity.TryGetField(Field, out _loadedField)) { return _loadedField; } } return null; }
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) { builder.Register<IEntityController>(ctx => { var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity); var output = new OutputContext(context, new Incrementer(context)); switch (output.Connection.Provider) { case "sqlserver": context.Debug("Registering sql server controller"); var initializer = process.Mode == "init" ? (IAction)new SqlEntityInitializer(output) : new NullInitializer(); return new SqlEntityController(output, initializer); default: context.Debug("Registering null controller"); return new NullEntityController(); } }).Named<IEntityController>(entity.Key); }
public string Create() { var schema = _schemaReader.Read(); // create a process based on the schema var process = new Process { Name = "JunkDrawer" }.WithDefaults(); process.Connections.Clear(); process.Connections.Add(schema.Connection.Clone()); process.Connections.Add(_cfg.Output()); process.Entities = schema.Entities; foreach (var entity in process.Entities) { entity.PrependProcessNameToOutputName = false; } if (!string.IsNullOrEmpty(_request.TableName)) { process.Entities.First().Alias = _request.TableName; } process.Mode = "init"; return new Root { Processes = new List<Process> { process } }.WithDefaults().Serialize(); }
public IProcessController Compose(string cfg, LogLevel logLevel = LogLevel.Debug) { var builder = new ContainerBuilder(); builder.RegisterModule(new ConfigurationModule(cfg, @"Files\Shorthand.xml")); var container = builder.Build(); Root = container.Resolve<Root>(); if (Root.Errors().Any()) { foreach (var error in Root.Errors()) { Console.Error.WriteLine(error); } throw new Exception("Configuration Error(s)"); } if (Root.Warnings().Any()) { foreach (var warning in Root.Warnings()) { Console.Error.WriteLine(warning); } } Process = Root.Processes.First(); builder = new ContainerBuilder(); builder.Register<IPipelineLogger>(ctx => new ConsoleLogger(logLevel)).SingleInstance(); builder.RegisterModule(new MapModule(Root)); builder.RegisterModule(new TemplateModule(Root)); builder.RegisterModule(new ActionModule(Root)); builder.RegisterModule(new EntityControlModule(Root)); builder.RegisterModule(new EntityPipelineModule(Root)); builder.RegisterModule(new EntityInputModule(Root)); builder.RegisterModule(new EntityOutputModule(Root)); builder.RegisterModule(new EntityMasterUpdateModule(Root)); builder.RegisterModule(new EntityDeleteModule(Root)); builder.RegisterModule(new EntityPipelineModule(Root)); builder.RegisterModule(new ProcessPipelineModule(Root)); builder.RegisterModule(new ProcessControlModule(Root)); container = builder.Build(); return container.ResolveNamed<IProcessController>(Process.Key); }
public PipelineContext( IPipelineLogger logger, Process process, Entity entity = null, Field field = null, Transform transform = null ) { ForLog = new object[5]; Logger = logger; Activity = PipelineActivity.Transform; Process = process; Entity = entity ?? process.GetValidatedOf<Entity>(e => { e.Name = string.Empty; }); Field = field ?? process.GetValidatedOf<Field>(f => { f.Name = string.Empty; }); Transform = transform ?? process.GetDefaultOf<Transform>(t => { t.Method = string.Empty; }); ForLog[0] = process.Name.PadRight(process.LogLimit, ' ').Left(process.LogLimit); ForLog[1] = Entity.Alias.PadRight(process.EntityLogLimit, ' ').Left(process.EntityLogLimit); ForLog[2] = ' '; ForLog[3] = Field.Alias.PadRight(process.FieldLogLimit, ' ').Left(process.FieldLogLimit); ForLog[4] = Transform.Method.PadRight(process.TransformLogLimit, ' ').Left(process.TransformLogLimit); }
private static Entity GetTestEntity() { var process = new Process(); return process.GetValidatedOf<Entity>(e => { e.Name = "e1"; e.Delete = true; e.Fields = new List<Field> { process.GetValidatedOf<Field>(f => { f.Index = 0; f.Name = "f1"; f.Type = "int"; f.PrimaryKey = true; }), process.GetValidatedOf<Field>(f => { f.Index = 1; f.Name = "f2"; f.Type = "string"; f.Length = "64"; }) }; }); }
public override void LoadEntity(ContainerBuilder builder, Process process, Entity entity) { var type = process.Pipeline == "defer" ? entity.Pipeline : process.Pipeline; builder.Register<IPipeline>((ctx) => { var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), process, entity); IPipeline pipeline; switch (type) { case "parallel.linq": context.Debug("Registering {0} pipeline.", type); pipeline = new ParallelPipeline(new DefaultPipeline(ctx.ResolveNamed<IEntityController>(entity.Key), context)); break; default: context.Debug("Registering linq pipeline.", type); pipeline = new DefaultPipeline(ctx.ResolveNamed<IEntityController>(entity.Key), context); break; } var provider = process.Connections.First(c => c.Name == "output").Provider; // extract pipeline.Register(ctx.ResolveNamed<IRead>(entity.Key)); // transform pipeline.Register(new DefaultTransform(context, context.GetAllEntityFields())); pipeline.Register(new TflHashCodeTransform(context)); pipeline.Register(TransformFactory.GetTransforms(ctx,process, entity, entity.GetAllFields().Where(f=>f.Transforms.Any()))); pipeline.Register(new StringTruncateTransfom(context)); if (provider == "sqlserver") { pipeline.Register(new MinDateTransform(context, new DateTime(1753, 1, 1))); } //load pipeline.Register(ctx.ResolveNamed<IWrite>(entity.Key)); pipeline.Register(ctx.ResolveNamed<IUpdate>(entity.Key)); return pipeline; }).Named<IPipeline>(entity.Key); }
public SqlCalculatedFieldUpdater(OutputContext context, Process original) { _context = context; _original = original; }
public SqlStarParametersReader(OutputContext output, Process process) { _output = output; _process = process; _rowCreator = new SqlRowCreator(output); }
protected override void RegisterProcess(ContainerBuilder builder, Process original) { // I need to create a new process with an entity with the appropriate fields // clone process, remove entities, and create entity needed for calcuted fields var calc = original.Clone() as Process; calc.Entities.Clear(); calc.CalculatedFields.Clear(); calc.Relationships.Clear(); var entity = original.GetDefaultOf<Entity>(e => { e.Name = "sys.Calc"; e.Connection = "output"; e.IsMaster = false; e.Fields = new List<Field> { original.GetDefaultOf<Field>(f => { f.Name = Constants.TflKey; f.Type = "int"; f.PrimaryKey = true; }) }; }); // Add fields that calculated fields depend on entity.Fields.AddRange(original.CalculatedFields .SelectMany(f => f.Transforms) .SelectMany(t => t.Parameters) .Where(p => !p.HasValue() && p.IsField(original)) .Select(p => p.AsField(original).Clone() as Field) ); entity.CalculatedFields.AddRange(original.CalculatedFields.Select(cf => cf.Clone() as Field)); calc.Entities.Add(entity); calc.ModifyKeys(); calc.ModifyIndexes(); // I need a process keyed pipeline builder.Register((ctx) => { var context = new PipelineContext(ctx.Resolve<IPipelineLogger>(), calc, entity); IPipeline pipeline; switch (original.Pipeline) { case "parallel.linq": context.Debug("Registering {0} pipeline.", original.Pipeline); pipeline = new ParallelPipeline(new DefaultPipeline(new NullEntityController(), context)); break; default: context.Debug("Registering linq pipeline.", original.Pipeline); pipeline = new DefaultPipeline(new NullEntityController(), context); break; } // register transforms pipeline.Register(new DefaultTransform(context, entity.CalculatedFields)); pipeline.Register(TransformFactory.GetTransforms(ctx, calc, entity, entity.CalculatedFields)); pipeline.Register(new StringTruncateTransfom(context)); // register input and output var outputContext = new OutputContext(context, new Incrementer(context)); switch (outputContext.Connection.Provider) { case "sqlserver": pipeline.Register(new SqlStarParametersReader(outputContext, original)); pipeline.Register(new SqlCalculatedFieldUpdater(outputContext, original)); pipeline.Register(new MinDateTransform(context, new DateTime(1753, 1, 1))); break; default: pipeline.Register(new NullReader(context)); pipeline.Register(new NullWriter(context)); break; } // no updater necessary pipeline.Register(new NullUpdater(context, false)); return pipeline; }).Named<IPipeline>(original.Key); }
public abstract void LoadEntity(ContainerBuilder builder, Process process, Entity entity);
protected abstract void RegisterProcess(ContainerBuilder builder, Process original);