public EntityInputKeysExtractDelta(Process process, Entity entity, AbstractConnection connection) : base(connection) { _entity = entity; _fields = _entity.PrimaryKey.WithInput().Aliases().ToArray(); _length = _fields.Length; EntityName = entity.Name; _entity.CheckForChanges(process, connection); if (!_entity.HasRows) { Debug("No data detected in {0}.", _entity.Alias); } if (_entity.BeginAndEndAreEqual()) { Debug("No changes detected in {0}.", _entity.Alias); } var keyQuery = _entity.CanDetectChanges(connection.IsDatabase) ? connection.KeyQuery(_entity) : connection.KeyAllQuery(_entity); var sql = _entity.HasRange ? connection.KeyRangeQuery(_entity) : keyQuery; Debug(sql); _sql = sql; }
public EntityJoinOperation(Process process, Relationship rel) : base(process) { _rel = rel; var rightFields = new HashSet<string>(rel.RightEntity.OutputFields().Aliases()); rightFields.ExceptWith(rel.LeftEntity.OutputFields().Aliases()); _fields = rightFields.ToArray(); }
private static void ProcessEntities(Process process) { var p = process; if (process.Entities.Count == 1) { new EntityProcess(p, process.Entities[0]).Execute(); } else { if (process.Parallel) { process.Entities.AsParallel().ForAll(e => new EntityProcess(p, e) { PipelineExecuter = e.PipelineThreading == PipelineThreading.SingleThreaded ? (IPipelineExecuter) new SingleThreadedPipelineExecuter() : (IPipelineExecuter) new ThreadPoolPipelineExecuter() }.Execute()); } else { foreach (var entityProcess in process.Entities.Select(entity => new EntityProcess(p, entity) { PipelineExecuter = entity.PipelineThreading == PipelineThreading.SingleThreaded ? (IPipelineExecuter) new SingleThreadedPipelineExecuter() : (IPipelineExecuter) new ThreadPoolPipelineExecuter() })) { entityProcess.Execute(); } } } }
private IOperation ComposeInputOperation(Process process, AbstractConnection connection) { if (connection.Schemas && _entity.Schema.Equals(string.Empty)) { _entity.Schema = connection.DefaultSchema; } if (_entity.HasSqlKeysOverride()) { return new SqlKeysOverrideOperation(_entity, connection); } if (!_entity.PrimaryKey.WithInput().Any()) { return new EmptyOperation(); } if (process.IsFirstRun || !_entity.CanDetectChanges(connection.IsDatabase)) { return connection.ExtractAllKeysFromInput(_process, _entity); } var operation = new EntityInputKeysExtractDelta(process, _entity, connection); if (operation.NeedsToRun()) { return operation; } return new EmptyOperation(); }
public IEnumerable <Row> Run(Process process) { var result = Enumerable.Empty <Row>(); if (!process.IsReady()) { return(result); } var timer = new Stopwatch(); timer.Start(); process.Setup(); var fileName = new FileInfo(Path.Combine(Common.GetTemporaryFolder(process.Name), "MetaData.xml")).FullName; var writer = new MetaDataWriter(process); File.WriteAllText(fileName, writer.Write(), Encoding.UTF8); System.Diagnostics.Process.Start(fileName); timer.Stop(); process.Logger.Info("Calculated metadata in {0}.", timer.Elapsed); process.Complete = true; return(result); }
public IEnumerable <Row> Run(Process process) { var result = Enumerable.Empty <Row>(); if (!process.IsReady()) { return(result); } var timer = new Stopwatch(); timer.Start(); process.Setup(); process.PerformActions(a => a.Before); new InitializationProcess(process).Execute(); new TemplateManager(process).Manage(); process.PerformActions(a => a.After); timer.Stop(); process.Logger.Info("Initialized output in {0}.", timer.Elapsed); process.Complete = true; return(result); }
public InitializationProcess(Process process) : base(process) { _process = process; process.OutputConnection.TflWriter.Initialize(process); foreach (var writer in process.OutputConnection.ViewWriters) { writer.Drop(process); } }
public EntityProcess(Process process, Entity entity) : base(process) { _process = process; _logger = process.Logger; _entity = entity; _collectors[STANDARD_OUTPUT] = new CollectorOperation(); PipelineExecuter = entity.PipelineThreading == PipelineThreading.SingleThreaded ? (AbstractPipelineExecuter)new SingleThreadedPipelineExecuter() : new ThreadPoolPipelineExecuter(); }
private static void ProcessMaster(Process process) { var updateMasterProcess = new UpdateMasterProcess(process) { PipelineExecuter = process.PipelineThreading == PipelineThreading.SingleThreaded ? (AbstractPipelineExecuter) new SingleThreadedPipelineExecuter() : new ThreadPoolPipelineExecuter() }; updateMasterProcess.Execute(); }
public EntityJoinAction(Process process, Entity entity) : base(process) { _hasVersion = entity.Version != null; _versionAlias = _hasVersion ? entity.Version.Alias : string.Empty; _versionSimpleType = _hasVersion ? entity.Version.SimpleType : string.Empty; _entityDelete = entity.Delete; _tflBatchId = entity.TflBatchId; _keys = entity.PrimaryKey.Aliases().ToArray(); _firstKey = _keys[0]; }
private static void ProcessDeletes(Process process) { var p = process; foreach (var entityDeleteProcess in process.Entities.Where(entity => entity.Delete).Select(entity => new EntityDeleteProcess(p, entity) { PipelineExecuter = entity.PipelineThreading == PipelineThreading.SingleThreaded ? (AbstractPipelineExecuter) new SingleThreadedPipelineExecuter() : new ThreadPoolPipelineExecuter() })) { entityDeleteProcess.Execute(); } }
public IEnumerable<Row> Run(Process process) { var result = Enumerable.Empty<Row>(); if (!process.IsReady()) { return result; } var timer = new Stopwatch(); timer.Start(); process.Setup(); process.IsFirstRun = process.MasterEntity == null || !process.OutputConnection.RecordsExist(process.MasterEntity); process.PerformActions(a => a.Before); if (!process.IsFirstRun) { ProcessDeletes(process); } ProcessEntities(process); if (process.StarEnabled && !process.OutputConnection.Is.Internal()) { ProcessMaster(process); } if (process.OutputConnection.Is.Internal()) { if (process.Relationships.Any()) { var collector = new CollectorOperation(); new MasterJoinProcess(process, ref collector).Execute(); process.Results = collector.Rows; } else { process.Results = process.MasterEntity == null ? Enumerable.Empty<Row>() : process.MasterEntity.Rows; } } else { process.Results = Enumerable.Empty<Row>(); } if (process.OutputConnection.Is.Internal()) { ProcessTransforms(process, new RowsOperation(process.Results)); } else { ProcessTransforms(process, new ParametersExtract(process)); } new TemplateManager(process).Manage(); process.PerformActions(a => a.After); timer.Stop(); process.Logger.Info( "Process affected {0} records in {1}.", process.Anything, timer.Elapsed); process.Complete = true; return process.Results; }
private static void ProcessTransforms(Process process, IOperation input) { if (process.CalculatedFields.Count <= 0) { return; } var transformProcess = new TransformProcess(process, input) { PipelineExecuter = process.PipelineThreading == PipelineThreading.SingleThreaded ? (AbstractPipelineExecuter) new SingleThreadedPipelineExecuter() : new ThreadPoolPipelineExecuter() }; transformProcess.Execute(); }
public string Handle(Process[] processes) { var xmlBuilder = new StringBuilder(); var xmlWriter = XmlWriter.Create(xmlBuilder, new XmlWriterSettings() { ConformanceLevel = ConformanceLevel.Fragment }); foreach (var rows in processes.Select(p => p.Results)) { foreach (var row in rows) { xmlWriter.WriteStartElement("add"); foreach (var column in row.Columns) { xmlWriter.WriteAttributeString(column, row[column].ToString()); } xmlWriter.WriteEndElement(); //add } } xmlWriter.Flush(); return xmlBuilder.ToString(); }
public string Handle(Process[] processes) { var xmlBuilder = new StringBuilder(); var xmlWriter = XmlWriter.Create(xmlBuilder, new XmlWriterSettings() { ConformanceLevel = ConformanceLevel.Fragment }); foreach (var process in processes) { if (process.Results.Any()) { xmlWriter.WriteStartElement("table"); var rows = process.Results.ToList(); var columns = rows.First().Columns.Where(c => !c.StartsWith("Tfl")).Select(c => c).ToArray(); var fields = process.OutputFields().Where(f => columns.Contains(f.Alias)).Select(f => new[] { f.Alias, string.IsNullOrEmpty(f.Label) ? f.Alias : f.Label, f.Raw.ToString(), f.SimpleType }).ToArray(); xmlWriter.WriteStartElement("thead"); xmlWriter.WriteStartElement("tr"); if (columns.Any()) { foreach (var field in fields) { xmlWriter.WriteElementString("th", field[1]); } } xmlWriter.WriteEndElement(); //tr xmlWriter.WriteEndElement(); //thead xmlWriter.WriteStartElement("tbody"); foreach (var row in process.Results) { xmlWriter.WriteStartElement("tr"); foreach (var field in fields) { xmlWriter.WriteStartElement("td"); if (field[2].Equals("True")) { xmlWriter.WriteRaw(row[field[0]].ToString()); } else { if (field[3].Equals("guid")) { xmlWriter.WriteValue(row[field[0]].ToString()); } else { xmlWriter.WriteValue(row[field[0]]); } } xmlWriter.WriteEndElement();//td } xmlWriter.WriteEndElement();//tr } xmlWriter.WriteEndElement(); //tbody xmlWriter.WriteEndElement(); //table } } xmlWriter.Flush(); return xmlBuilder.ToString(); }
public TestWithProcess() { var cfg = new TflRoot(File.ReadAllText(@"TestFiles\Test.xml"), null); foreach (var problem in cfg.Errors()) { Console.WriteLine(problem); } _process = ProcessFactory.CreateSingle(cfg.Processes[0], new TestLogger()); _entityKeysExtract = new Mock<IOperation>(); _entityKeysExtract.Setup(foo => foo.Execute(It.IsAny<IEnumerable<Row>>())).Returns(new List<Row> { new Row { {"OrderDetailKey", 1} }, new Row { {"OrderDetailKey", 2} }, new Row { {"OrderDetailKey", 3} }, new Row { {"OrderDetailKey", 4} } }); }
public string Handle(Process[] processes) { var sw = new StringWriter(); var writer = new JsonTextWriter(sw); foreach (var process in processes) { writer.WriteStartArray(); foreach (var row in process.Results) { writer.WriteStartArray(); foreach (var alias in process.OutputFields().Aliases()) { writer.WriteValue(row[alias]); } writer.WriteEndArray(); } writer.WriteEndArray(); } writer.Flush(); return sw.ToString(); }
public Template(Process process, TflTemplate element, Contents contents) { Parameters = new Parameters.Parameters(new DefaultFactory(process.Logger)); Cache = element.Cache; Enabled = element.Enabled; Engine = element.Engine; Name = element.Name; ContentType = element.ContentType.Equals("raw") ? Encoding.Raw : Encoding.Html; Contents = contents; _process = process; _renderedTemplateFile = GetFileName(RENDERED_TEMPLATE_CACHE_FOLDER); _renderedTemplateContentExists = TryRead(_renderedTemplateFile, out _renderedTemplateContent); _templateFile = GetFileName(TEMPLATE_CACHE_FOLDER); _templateContentExists = TryRead(_templateFile, out _templateContent); }
public EntityKeysPartial(Process process, Entity entity) : base(process) { _process = process; _entity = entity; if (entity.Input.Count == 1) { var connection = entity.Input.First().Connection; if (connection.IsDatabase && !entity.HasSqlOverride()) { Register(ComposeInputOperation(process, connection)); } } else { var union = new ParallelUnionAllOperation(); foreach (var input in entity.Input) { if (input.Connection.IsDatabase && !_entity.HasSqlOverride()) { union.Add(ComposeInputOperation(process, input.Connection)); } } Register(union); } Register(new EntityKeysDistinct(_entity)); }
public IEnumerable<Row> Run(Process process) { var result = Enumerable.Empty<Row>(); if (!process.IsReady()) { return result; } var timer = new Stopwatch(); timer.Start(); process.Setup(); var fileName = new FileInfo(Path.Combine(Common.GetTemporaryFolder(process.Name), "MetaData.xml")).FullName; var writer = new MetaDataWriter(process); File.WriteAllText(fileName, writer.Write(), Encoding.UTF8); System.Diagnostics.Process.Start(fileName); timer.Stop(); process.Logger.Info( "Calculated metadata in {0}.", timer.Elapsed); process.Complete = true; return result; }
public TemplateAction(Process process, string template, TflAction action) { _process = process; ProcessName = process.Name; Action = action.Action; File = action.File; Method = action.Method; Url = action.Url; TemplateName = template; From = action.From; Arguments = action.Arguments; Bcc = action.Bcc; Cc = action.Cc; Html = action.Html; Subject = action.Subject; To = action.To; Body = action.Body; Modes = action.GetModes(); Before = action.Before; After = action.After; OldValue = action.OldValue; NewValue = action.NewValue; Command = action.Command; Timeout = action.TimeOut; ActionHandlers = new Dictionary<string, TemplateActionHandler>() { {"copy", new TemplateActionCopy(_process)}, {"open", new TemplateActionOpen(_process.Logger)}, {"run", new TemplateActionRun(_process.Logger)}, {"web", new TemplateActionWeb(_process.Logger)}, {"tfl", new TemplateActionTfl(_process.Logger)}, {"exec", new TemplateActionExecute(_process)}, {"execute", new TemplateActionExecute(_process)}, {"mail", new TemplateActionMail(_process.Logger)}, {"replace", new TemplateActionReplace(_process.Logger)}, {"dostounix", new TemplateActionReplace("\r\n", "\n")}, {"unixtodos", new TemplateActionReplace("\n","\r\n")} }; }
public IEnumerable<Row> Run(Process process) { var result = Enumerable.Empty<Row>(); if (!process.IsReady()) { return result; } var timer = new Stopwatch(); timer.Start(); process.Setup(); process.PerformActions(a => a.Before); new InitializationProcess(process).Execute(); new TemplateManager(process).Manage(); process.PerformActions(a => a.After); timer.Stop(); process.Logger.Info("Initialized output in {0}.", timer.Elapsed); process.Complete = true; return result; }
public EntityDeleteProcess(Process process, Entity entity) : base(process) { _process = process; _logger = process.Logger; _entity = entity; }
public IEnumerable <Row> Run(Process process) { var result = Enumerable.Empty <Row>(); if (!process.IsReady()) { return(result); } var timer = new Stopwatch(); timer.Start(); process.Setup(); process.IsFirstRun = process.MasterEntity == null || !process.OutputConnection.RecordsExist(process.MasterEntity); process.PerformActions(a => a.Before); if (!process.IsFirstRun) { ProcessDeletes(process); } ProcessEntities(process); if (process.StarEnabled && !process.OutputConnection.Is.Internal()) { ProcessMaster(process); } if (process.OutputConnection.Is.Internal()) { if (process.Relationships.Any()) { var collector = new CollectorOperation(); new MasterJoinProcess(process, ref collector).Execute(); process.Results = collector.Rows; } else { process.Results = process.MasterEntity == null?Enumerable.Empty <Row>() : process.MasterEntity.Rows; } } else { process.Results = Enumerable.Empty <Row>(); } if (process.OutputConnection.Is.Internal()) { ProcessTransforms(process, new RowsOperation(process.Results)); } else { ProcessTransforms(process, new ParametersExtract(process)); } new TemplateManager(process).Manage(); process.PerformActions(a => a.After); timer.Stop(); process.Logger.Info("Process affected {0} records in {1}.", process.Anything, timer.Elapsed); process.Complete = true; return(process.Results); }
public ParametersExtract(Process process) : base(process.OutputConnection) { _process = process; UseTransaction = false; _keys = new HashSet<string>(_process.Parameters.ToEnumerable().Where(p => !p.Value.HasValue()).Select(p => p.Key)); _keys.IntersectWith(process.OutputFields().Select(f=>f.Alias)); }
public MasterEntityIndex(Process process) { _process = process; Name = "MasterEntityIndex"; }
public EntityDetectDeletes(Process process, Entity entity) : base(process) { _keys = entity.PrimaryKey.Aliases().ToArray(); _firstKey = _keys[0]; }
public EntityRelationshipLoader(Process process) { _process = process; }
public ProcessTransformParametersReader(Process process, DefaultFactory defaultFactory) { _process = process; _defaultFactory = defaultFactory; }
public EntityConfigurationLoader(Process process) { _process = process; }
protected JoinOperation(Process process) { left = new PartialProcessOperation(process); right = new PartialProcessOperation(process); }
public FieldReader(Process process, Entity entity, bool usePrefix = true) { _process = process; _usePrefix = usePrefix; _entity = entity; }
public MasterEntityIndexBuilder(Process process) { _process = process; _logger = process.Logger; }
public Process Read() { _process = new Process(_configuration.Name, _logger) { Configuration = _configuration, Options = _options, TemplateContentType = _configuration.TemplateContentType.Equals("raw") ? Encoding.Raw : Encoding.Html, Enabled = _configuration.Enabled, FileInspectionRequest = _configuration.FileInspection.Count == 0 ? new FileInspectionRequest(_configuration.Name) : _configuration.FileInspection[0].GetInspectionRequest(_configuration.Name), Star = _configuration.Star, View = _configuration.View, Mode = _configuration.Mode, StarEnabled = _configuration.StarEnabled, ViewEnabled = _configuration.ViewEnabled, TimeZone = string.IsNullOrEmpty(_configuration.TimeZone) ? TimeZoneInfo.Local.Id : _configuration.TimeZone, PipelineThreading = (PipelineThreading)Enum.Parse(typeof(PipelineThreading), _configuration.PipelineThreading, true), Parallel = _configuration.Parallel, Runner = _runner }; //shared across the process _process.Connections.AddRange(_configuration.Connections); _process.OutputConnection = _process.Connections.Output().Connection; _process.DataSets = GetDataSets(_configuration); _process.Scripts = new ScriptReader(_configuration.Scripts, _logger).Read(); _process.Actions = new ActionReader(_process).Read(_configuration.Actions); _process.Templates = new TemplateReader(_process, _configuration.Templates).Read(); _process.SearchTypes = new SearchTypeReader(_configuration.SearchTypes).Read(); new MapLoader(_process, _configuration.Maps).Load(); //these depend on the shared process properties new EntitiesLoader(_process, _configuration.Entities).Load(); new OperationsLoader(_process, _configuration.Entities).Load(); _process.Relationships = new RelationshipsReader(_process, _configuration.Relationships).Read(); new ProcessOperationsLoader(_process, _configuration.CalculatedFields).Load(); new EntityRelationshipLoader(_process).Load(); return _process; }
public TemplateActionExecute(Process process) { _process = process; }
public UpdateMasterProcess(Process process) : base(process) { _process = process; }
public TransformalizeResponse() { Processes = new Process[0]; Log = new List<string[]>(); }
public EntityActionFilter(Process process, Entity entity, EntityAction entityAction) { _process = process; _entity = entity; _entityAction = entityAction; }