public EntityInputKeysExtractDelta(Process process, Entity entity, AbstractConnection connection)
: base(connection) {
_entity = entity;
_fields = _entity.PrimaryKey.WithInput().Aliases().ToArray();
_length = _fields.Length;
EntityName = entity.Name;
_entity.CheckForChanges(process, connection);
if (!_entity.HasRows) {
Debug("No data detected in {0}.", _entity.Alias);
}
if (_entity.BeginAndEndAreEqual()) {
Debug("No changes detected in {0}.", _entity.Alias);
}
var keyQuery = _entity.CanDetectChanges(connection.IsDatabase)
? connection.KeyQuery(_entity)
: connection.KeyAllQuery(_entity);
var sql = _entity.HasRange ?
connection.KeyRangeQuery(_entity) :
keyQuery;
Debug(sql);
_sql = sql;
}
public EntityJoinOperation(Process process, Relationship rel)
: base(process) {
_rel = rel;
var rightFields = new HashSet<string>(rel.RightEntity.OutputFields().Aliases());
rightFields.ExceptWith(rel.LeftEntity.OutputFields().Aliases());
_fields = rightFields.ToArray();
}
private static void ProcessEntities(Process process)
{
var p = process;
if (process.Entities.Count == 1)
{
new EntityProcess(p, process.Entities[0]).Execute();
}
else
{
if (process.Parallel)
{
process.Entities.AsParallel().ForAll(e => new EntityProcess(p, e)
{
PipelineExecuter = e.PipelineThreading == PipelineThreading.SingleThreaded ?
(IPipelineExecuter) new SingleThreadedPipelineExecuter() :
(IPipelineExecuter) new ThreadPoolPipelineExecuter()
}.Execute());
}
else
{
foreach (var entityProcess in process.Entities.Select(entity => new EntityProcess(p, entity)
{
PipelineExecuter = entity.PipelineThreading == PipelineThreading.SingleThreaded
? (IPipelineExecuter) new SingleThreadedPipelineExecuter()
: (IPipelineExecuter) new ThreadPoolPipelineExecuter()
}))
{
entityProcess.Execute();
}
}
}
}
private IOperation ComposeInputOperation(Process process, AbstractConnection connection) {
if (connection.Schemas && _entity.Schema.Equals(string.Empty)) {
_entity.Schema = connection.DefaultSchema;
}
if (_entity.HasSqlKeysOverride()) {
return new SqlKeysOverrideOperation(_entity, connection);
}
if (!_entity.PrimaryKey.WithInput().Any()) {
return new EmptyOperation();
}
if (process.IsFirstRun || !_entity.CanDetectChanges(connection.IsDatabase)) {
return connection.ExtractAllKeysFromInput(_process, _entity);
}
var operation = new EntityInputKeysExtractDelta(process, _entity, connection);
if (operation.NeedsToRun()) {
return operation;
}
return new EmptyOperation();
}
public IEnumerable <Row> Run(Process process)
{
var result = Enumerable.Empty <Row>();
if (!process.IsReady())
{
return(result);
}
var timer = new Stopwatch();
timer.Start();
process.Setup();
var fileName = new FileInfo(Path.Combine(Common.GetTemporaryFolder(process.Name), "MetaData.xml")).FullName;
var writer = new MetaDataWriter(process);
File.WriteAllText(fileName, writer.Write(), Encoding.UTF8);
System.Diagnostics.Process.Start(fileName);
timer.Stop();
process.Logger.Info("Calculated metadata in {0}.", timer.Elapsed);
process.Complete = true;
return(result);
}
public IEnumerable <Row> Run(Process process)
{
var result = Enumerable.Empty <Row>();
if (!process.IsReady())
{
return(result);
}
var timer = new Stopwatch();
timer.Start();
process.Setup();
process.PerformActions(a => a.Before);
new InitializationProcess(process).Execute();
new TemplateManager(process).Manage();
process.PerformActions(a => a.After);
timer.Stop();
process.Logger.Info("Initialized output in {0}.", timer.Elapsed);
process.Complete = true;
return(result);
}
public InitializationProcess(Process process)
: base(process) {
_process = process;
process.OutputConnection.TflWriter.Initialize(process);
foreach (var writer in process.OutputConnection.ViewWriters) {
writer.Drop(process);
}
}
public EntityProcess(Process process, Entity entity)
: base(process) {
_process = process;
_logger = process.Logger;
_entity = entity;
_collectors[STANDARD_OUTPUT] = new CollectorOperation();
PipelineExecuter = entity.PipelineThreading == PipelineThreading.SingleThreaded ? (AbstractPipelineExecuter)new SingleThreadedPipelineExecuter() : new ThreadPoolPipelineExecuter();
}
private static void ProcessMaster(Process process)
{
var updateMasterProcess = new UpdateMasterProcess(process)
{
PipelineExecuter = process.PipelineThreading == PipelineThreading.SingleThreaded ? (AbstractPipelineExecuter) new SingleThreadedPipelineExecuter() : new ThreadPoolPipelineExecuter()
};
updateMasterProcess.Execute();
}
public EntityJoinAction(Process process, Entity entity)
: base(process) {
_hasVersion = entity.Version != null;
_versionAlias = _hasVersion ? entity.Version.Alias : string.Empty;
_versionSimpleType = _hasVersion ? entity.Version.SimpleType : string.Empty;
_entityDelete = entity.Delete;
_tflBatchId = entity.TflBatchId;
_keys = entity.PrimaryKey.Aliases().ToArray();
_firstKey = _keys[0];
}
private static void ProcessDeletes(Process process)
{
var p = process;
foreach (var entityDeleteProcess in process.Entities.Where(entity => entity.Delete).Select(entity => new EntityDeleteProcess(p, entity)
{
PipelineExecuter = entity.PipelineThreading == PipelineThreading.SingleThreaded ? (AbstractPipelineExecuter) new SingleThreadedPipelineExecuter() : new ThreadPoolPipelineExecuter()
}))
{
entityDeleteProcess.Execute();
}
}
public IEnumerable<Row> Run(Process process) {
var result = Enumerable.Empty<Row>();
if (!process.IsReady()) {
return result;
}
var timer = new Stopwatch();
timer.Start();
process.Setup();
process.IsFirstRun = process.MasterEntity == null || !process.OutputConnection.RecordsExist(process.MasterEntity);
process.PerformActions(a => a.Before);
if (!process.IsFirstRun) {
ProcessDeletes(process);
}
ProcessEntities(process);
if (process.StarEnabled && !process.OutputConnection.Is.Internal()) {
ProcessMaster(process);
}
if (process.OutputConnection.Is.Internal()) {
if (process.Relationships.Any()) {
var collector = new CollectorOperation();
new MasterJoinProcess(process, ref collector).Execute();
process.Results = collector.Rows;
} else {
process.Results = process.MasterEntity == null ? Enumerable.Empty<Row>() : process.MasterEntity.Rows;
}
} else {
process.Results = Enumerable.Empty<Row>();
}
if (process.OutputConnection.Is.Internal()) {
ProcessTransforms(process, new RowsOperation(process.Results));
} else {
ProcessTransforms(process, new ParametersExtract(process));
}
new TemplateManager(process).Manage();
process.PerformActions(a => a.After);
timer.Stop();
process.Logger.Info( "Process affected {0} records in {1}.", process.Anything, timer.Elapsed);
process.Complete = true;
return process.Results;
}
private static void ProcessTransforms(Process process, IOperation input)
{
if (process.CalculatedFields.Count <= 0)
{
return;
}
var transformProcess = new TransformProcess(process, input)
{
PipelineExecuter = process.PipelineThreading == PipelineThreading.SingleThreaded ? (AbstractPipelineExecuter) new SingleThreadedPipelineExecuter() : new ThreadPoolPipelineExecuter()
};
transformProcess.Execute();
}
public string Handle(Process[] processes) {
var xmlBuilder = new StringBuilder();
var xmlWriter = XmlWriter.Create(xmlBuilder, new XmlWriterSettings() { ConformanceLevel = ConformanceLevel.Fragment });
foreach (var rows in processes.Select(p => p.Results)) {
foreach (var row in rows) {
xmlWriter.WriteStartElement("add");
foreach (var column in row.Columns) {
xmlWriter.WriteAttributeString(column, row[column].ToString());
}
xmlWriter.WriteEndElement(); //add
}
}
xmlWriter.Flush();
return xmlBuilder.ToString();
}
public string Handle(Process[] processes) {
var xmlBuilder = new StringBuilder();
var xmlWriter = XmlWriter.Create(xmlBuilder, new XmlWriterSettings() { ConformanceLevel = ConformanceLevel.Fragment });
foreach (var process in processes) {
if (process.Results.Any()) {
xmlWriter.WriteStartElement("table");
var rows = process.Results.ToList();
var columns = rows.First().Columns.Where(c => !c.StartsWith("Tfl")).Select(c => c).ToArray();
var fields = process.OutputFields().Where(f => columns.Contains(f.Alias)).Select(f => new[] { f.Alias, string.IsNullOrEmpty(f.Label) ? f.Alias : f.Label, f.Raw.ToString(), f.SimpleType }).ToArray();
xmlWriter.WriteStartElement("thead");
xmlWriter.WriteStartElement("tr");
if (columns.Any()) {
foreach (var field in fields) {
xmlWriter.WriteElementString("th", field[1]);
}
}
xmlWriter.WriteEndElement(); //tr
xmlWriter.WriteEndElement(); //thead
xmlWriter.WriteStartElement("tbody");
foreach (var row in process.Results) {
xmlWriter.WriteStartElement("tr");
foreach (var field in fields) {
xmlWriter.WriteStartElement("td");
if (field[2].Equals("True")) {
xmlWriter.WriteRaw(row[field[0]].ToString());
} else {
if (field[3].Equals("guid")) {
xmlWriter.WriteValue(row[field[0]].ToString());
} else {
xmlWriter.WriteValue(row[field[0]]);
}
}
xmlWriter.WriteEndElement();//td
}
xmlWriter.WriteEndElement();//tr
}
xmlWriter.WriteEndElement(); //tbody
xmlWriter.WriteEndElement(); //table
}
}
xmlWriter.Flush();
return xmlBuilder.ToString();
}
public TestWithProcess() {
var cfg = new TflRoot(File.ReadAllText(@"TestFiles\Test.xml"), null);
foreach (var problem in cfg.Errors()) {
Console.WriteLine(problem);
}
_process = ProcessFactory.CreateSingle(cfg.Processes[0], new TestLogger());
_entityKeysExtract = new Mock<IOperation>();
_entityKeysExtract.Setup(foo => foo.Execute(It.IsAny<IEnumerable<Row>>())).Returns(new List<Row> {
new Row { {"OrderDetailKey", 1} },
new Row { {"OrderDetailKey", 2} },
new Row { {"OrderDetailKey", 3} },
new Row { {"OrderDetailKey", 4} }
});
}
public string Handle(Process[] processes) {
var sw = new StringWriter();
var writer = new JsonTextWriter(sw);
foreach (var process in processes) {
writer.WriteStartArray();
foreach (var row in process.Results) {
writer.WriteStartArray();
foreach (var alias in process.OutputFields().Aliases()) {
writer.WriteValue(row[alias]);
}
writer.WriteEndArray();
}
writer.WriteEndArray();
}
writer.Flush();
return sw.ToString();
}
public Template(Process process, TflTemplate element, Contents contents) {
Parameters = new Parameters.Parameters(new DefaultFactory(process.Logger));
Cache = element.Cache;
Enabled = element.Enabled;
Engine = element.Engine;
Name = element.Name;
ContentType = element.ContentType.Equals("raw") ? Encoding.Raw : Encoding.Html;
Contents = contents;
_process = process;
_renderedTemplateFile = GetFileName(RENDERED_TEMPLATE_CACHE_FOLDER);
_renderedTemplateContentExists = TryRead(_renderedTemplateFile, out _renderedTemplateContent);
_templateFile = GetFileName(TEMPLATE_CACHE_FOLDER);
_templateContentExists = TryRead(_templateFile, out _templateContent);
}
public EntityKeysPartial(Process process, Entity entity) : base(process) {
_process = process;
_entity = entity;
if (entity.Input.Count == 1) {
var connection = entity.Input.First().Connection;
if (connection.IsDatabase && !entity.HasSqlOverride()) {
Register(ComposeInputOperation(process, connection));
}
} else {
var union = new ParallelUnionAllOperation();
foreach (var input in entity.Input) {
if (input.Connection.IsDatabase && !_entity.HasSqlOverride()) {
union.Add(ComposeInputOperation(process, input.Connection));
}
}
Register(union);
}
Register(new EntityKeysDistinct(_entity));
}
public IEnumerable<Row> Run(Process process) {
var result = Enumerable.Empty<Row>();
if (!process.IsReady()) {
return result;
}
var timer = new Stopwatch();
timer.Start();
process.Setup();
var fileName = new FileInfo(Path.Combine(Common.GetTemporaryFolder(process.Name), "MetaData.xml")).FullName;
var writer = new MetaDataWriter(process);
File.WriteAllText(fileName, writer.Write(), Encoding.UTF8);
System.Diagnostics.Process.Start(fileName);
timer.Stop();
process.Logger.Info( "Calculated metadata in {0}.", timer.Elapsed);
process.Complete = true;
return result;
}
public TemplateAction(Process process, string template, TflAction action) {
_process = process;
ProcessName = process.Name;
Action = action.Action;
File = action.File;
Method = action.Method;
Url = action.Url;
TemplateName = template;
From = action.From;
Arguments = action.Arguments;
Bcc = action.Bcc;
Cc = action.Cc;
Html = action.Html;
Subject = action.Subject;
To = action.To;
Body = action.Body;
Modes = action.GetModes();
Before = action.Before;
After = action.After;
OldValue = action.OldValue;
NewValue = action.NewValue;
Command = action.Command;
Timeout = action.TimeOut;
ActionHandlers = new Dictionary<string, TemplateActionHandler>() {
{"copy", new TemplateActionCopy(_process)},
{"open", new TemplateActionOpen(_process.Logger)},
{"run", new TemplateActionRun(_process.Logger)},
{"web", new TemplateActionWeb(_process.Logger)},
{"tfl", new TemplateActionTfl(_process.Logger)},
{"exec", new TemplateActionExecute(_process)},
{"execute", new TemplateActionExecute(_process)},
{"mail", new TemplateActionMail(_process.Logger)},
{"replace", new TemplateActionReplace(_process.Logger)},
{"dostounix", new TemplateActionReplace("\r\n", "\n")},
{"unixtodos", new TemplateActionReplace("\n","\r\n")}
};
}
public IEnumerable<Row> Run(Process process) {
var result = Enumerable.Empty<Row>();
if (!process.IsReady()) {
return result;
}
var timer = new Stopwatch();
timer.Start();
process.Setup();
process.PerformActions(a => a.Before);
new InitializationProcess(process).Execute();
new TemplateManager(process).Manage();
process.PerformActions(a => a.After);
timer.Stop();
process.Logger.Info("Initialized output in {0}.", timer.Elapsed);
process.Complete = true;
return result;
}
public EntityDeleteProcess(Process process, Entity entity)
: base(process) {
_process = process;
_logger = process.Logger;
_entity = entity;
}
public IEnumerable <Row> Run(Process process)
{
var result = Enumerable.Empty <Row>();
if (!process.IsReady())
{
return(result);
}
var timer = new Stopwatch();
timer.Start();
process.Setup();
process.IsFirstRun = process.MasterEntity == null || !process.OutputConnection.RecordsExist(process.MasterEntity);
process.PerformActions(a => a.Before);
if (!process.IsFirstRun)
{
ProcessDeletes(process);
}
ProcessEntities(process);
if (process.StarEnabled && !process.OutputConnection.Is.Internal())
{
ProcessMaster(process);
}
if (process.OutputConnection.Is.Internal())
{
if (process.Relationships.Any())
{
var collector = new CollectorOperation();
new MasterJoinProcess(process, ref collector).Execute();
process.Results = collector.Rows;
}
else
{
process.Results = process.MasterEntity == null?Enumerable.Empty <Row>() : process.MasterEntity.Rows;
}
}
else
{
process.Results = Enumerable.Empty <Row>();
}
if (process.OutputConnection.Is.Internal())
{
ProcessTransforms(process, new RowsOperation(process.Results));
}
else
{
ProcessTransforms(process, new ParametersExtract(process));
}
new TemplateManager(process).Manage();
process.PerformActions(a => a.After);
timer.Stop();
process.Logger.Info("Process affected {0} records in {1}.", process.Anything, timer.Elapsed);
process.Complete = true;
return(process.Results);
}
public ParametersExtract(Process process) : base(process.OutputConnection) {
_process = process;
UseTransaction = false;
_keys = new HashSet<string>(_process.Parameters.ToEnumerable().Where(p => !p.Value.HasValue()).Select(p => p.Key));
_keys.IntersectWith(process.OutputFields().Select(f=>f.Alias));
}
public MasterEntityIndex(Process process) {
_process = process;
Name = "MasterEntityIndex";
}
public EntityDetectDeletes(Process process, Entity entity)
: base(process) {
_keys = entity.PrimaryKey.Aliases().ToArray();
_firstKey = _keys[0];
}
public EntityRelationshipLoader(Process process) {
_process = process;
}
public ProcessTransformParametersReader(Process process, DefaultFactory defaultFactory)
{
_process = process;
_defaultFactory = defaultFactory;
}
public EntityConfigurationLoader(Process process) {
_process = process;
}
protected JoinOperation(Process process) {
left = new PartialProcessOperation(process);
right = new PartialProcessOperation(process);
}
public FieldReader(Process process, Entity entity, bool usePrefix = true) {
_process = process;
_usePrefix = usePrefix;
_entity = entity;
}
public MasterEntityIndexBuilder(Process process)
{
_process = process;
_logger = process.Logger;
}
public Process Read() {
_process = new Process(_configuration.Name, _logger) {
Configuration = _configuration,
Options = _options,
TemplateContentType = _configuration.TemplateContentType.Equals("raw") ? Encoding.Raw : Encoding.Html,
Enabled = _configuration.Enabled,
FileInspectionRequest = _configuration.FileInspection.Count == 0 ? new FileInspectionRequest(_configuration.Name) : _configuration.FileInspection[0].GetInspectionRequest(_configuration.Name),
Star = _configuration.Star,
View = _configuration.View,
Mode = _configuration.Mode,
StarEnabled = _configuration.StarEnabled,
ViewEnabled = _configuration.ViewEnabled,
TimeZone = string.IsNullOrEmpty(_configuration.TimeZone) ? TimeZoneInfo.Local.Id : _configuration.TimeZone,
PipelineThreading = (PipelineThreading)Enum.Parse(typeof(PipelineThreading), _configuration.PipelineThreading, true),
Parallel = _configuration.Parallel,
Runner = _runner
};
//shared across the process
_process.Connections.AddRange(_configuration.Connections);
_process.OutputConnection = _process.Connections.Output().Connection;
_process.DataSets = GetDataSets(_configuration);
_process.Scripts = new ScriptReader(_configuration.Scripts, _logger).Read();
_process.Actions = new ActionReader(_process).Read(_configuration.Actions);
_process.Templates = new TemplateReader(_process, _configuration.Templates).Read();
_process.SearchTypes = new SearchTypeReader(_configuration.SearchTypes).Read();
new MapLoader(_process, _configuration.Maps).Load();
//these depend on the shared process properties
new EntitiesLoader(_process, _configuration.Entities).Load();
new OperationsLoader(_process, _configuration.Entities).Load();
_process.Relationships = new RelationshipsReader(_process, _configuration.Relationships).Read();
new ProcessOperationsLoader(_process, _configuration.CalculatedFields).Load();
new EntityRelationshipLoader(_process).Load();
return _process;
}
public TemplateActionExecute(Process process) {
_process = process;
}
public UpdateMasterProcess(Process process)
: base(process) {
_process = process;
}
public TransformalizeResponse() {
Processes = new Process[0];
Log = new List<string[]>();
}
public EntityActionFilter(Process process, Entity entity, EntityAction entityAction) {
_process = process;
_entity = entity;
_entityAction = entityAction;
}