public ActionResult Load(int id)
{
Response.AddHeader("Access-Control-Allow-Origin", "*");
ConfigurationPart part;
ApiRequest request;
foreach (var rejection in _apiService.Rejections(id, out request, out part))
{
return(rejection.ContentResult(_query["format"], _query["flavor"]));
}
request.RequestType = ApiRequestType.Execute;
var root = new TflRoot(part.Configuration, _query);
TransformConfigurationForLoad(root);
var errors = root.Errors();
if (errors.Any())
{
request.Status = 500;
request.Message = string.Join(Environment.NewLine, errors);
return(new ApiResponse(request, root.ToString(), new TransformalizeResponse()).ContentResult(_query["format"], _query["flavor"]));
}
var transformalizeRequest = new TransformalizeRequest(part, _query, null, Logger, root);
return(Run(request, transformalizeRequest)
.ContentResult(_query["format"], _query["flavor"]));
}
public List <TflProcess> Read(Dictionary <string, string> parameters)
{
var shouldThrow = false;
var content = _contentsReader.Read(_resource).Content;
var cfg = new TflRoot(content, parameters);
if (cfg.Response.Any())
{
foreach (var response in cfg.Response.Where(response => response.Status != (short)200))
{
_logger.Warn("API at {0} responded with {1} {2}.", _resource, response.Status, response.Message);
}
}
foreach (var warning in cfg.Warnings())
{
_logger.Warn(warning);
}
foreach (var error in cfg.Errors())
{
shouldThrow = true;
_logger.Error(error);
}
if (shouldThrow)
{
throw new TransformalizeException(_logger, string.Join(Environment.NewLine, cfg.Errors()));
}
return(cfg.Processes);
}
public void TestEnvironmentParameters()
{
const string xml = @"<transformalize>
<environments>
<add name=""e1"">
<parameters>
<add name=""t1"" value=""v1"" />
<add name=""t2"" value=""v2"" />
<add name=""t3"" value=""v3"" />
</parameters>
</add>
</environments>
<processes>
<add name=""test1"">
<actions>
<add action=""@(t1)"" />
</actions>
</add>
<add name=""test2"">
<actions>
<add action=""@(t2)"" />
<add action=""@(t3)"" />
</actions>
</add>
</processes>
</transformalize>";
var processes = new TflRoot(xml, null).Processes;
Assert.AreEqual("v1", processes[0].Actions[0].Action);
Assert.AreEqual("v2", processes[1].Actions[0].Action);
Assert.AreEqual("v3", processes[1].Actions[1].Action);
}
public TestWithProcess()
{
var cfg = new TflRoot(File.ReadAllText(@"TestFiles\Test.xml"), null);
foreach (var problem in cfg.Errors())
{
Console.WriteLine(problem);
}
_process = ProcessFactory.CreateSingle(cfg.Processes[0], new TestLogger());
_entityKeysExtract = new Mock <IOperation>();
_entityKeysExtract.Setup(foo => foo.Execute(It.IsAny <IEnumerable <Row> >())).Returns(new List <Row> {
new Row {
{ "OrderDetailKey", 1 }
},
new Row {
{ "OrderDetailKey", 2 }
},
new Row {
{ "OrderDetailKey", 3 }
},
new Row {
{ "OrderDetailKey", 4 }
}
});
}
protected static void TransformConfigurationForDownload(TflRoot root, Dictionary <string, string> query)
{
ReverseConfiguration(root);
var output = root.Processes[0].Connections.First(c => c.Name == "output");
output.Provider = "file";
output.File = query.ContainsKey("OutputFile") ? query["OutputFile"] : string.Empty;
}
public void TestError()
{
var cfg = new TflRoot(File.ReadAllText(@"C:\Temp\test.xml"), null);
Assert.AreEqual(0, cfg.Errors().Length);
var process = ProcessFactory.CreateSingle(@"C:\Temp\test.xml", new TestLogger());
Assert.IsNotNull(process);
process.ExecuteScaler();
}
protected static void ReverseConfiguration(TflRoot root)
{
var process = root.Processes[0];
process.StarEnabled = false;
//swap input and output
process.Connections.First(c => c.Name.Equals("output")).Name = "temp";
process.Connections.First(c => c.Name.Equals("input")).Name = "output";
process.Connections.First(c => c.Name.Equals("temp")).Name = "input";
var entity = process.Entities[0];
entity.Name = GetEntityOutputName(entity, process.Name);
//if no primary key is set, make all the fields a composite primary key
if (!entity.Fields.Any(f => f.PrimaryKey))
{
foreach (var field in entity.Fields)
{
field.PrimaryKey = true;
}
}
//aliases are used to write to output, so swap when reversing
foreach (var field in entity.Fields)
{
var temp = field.Name;
field.Name = field.Alias;
field.Alias = temp;
}
entity.DetectChanges = false;
if (!entity.Delete)
{
return;
}
entity.Delete = false;
entity.Filter.Add(entity.GetDefaultOf <TflFilter>(f => {
f.Left = "TflDeleted";
f.Right = "0";
}));
entity.Fields.Add(entity.GetDefaultOf <TflField>(f => {
f.Name = "TflDeleted";
f.Type = "bool";
f.Output = false;
f.Label = "Deleted";
}));
}
public void Test2()
{
var xml = @"
<cfg>
<processes>
<add name='test'>
<data-sets>
<add name='one'>
<rows>
<add f1='1' f2='1' f3='2001-01-01' />
<add f1='2' f2='2' f3='2002-02-02' />
<add f1='3' f2='3' f3='2003-03-03' />
</rows>
</add>
</data-sets>
<connections>
<add name='input' provider='internal' />
<add name='output' provider='internal' />
</connections>
<entities>
<add name='one'>
<fields>
<add name='f1' primary-key='true' />
<add name='f2' type='int' />
<add name='f3' type='datetime' />
</fields>
</add>
</entities>
</add>
</processes>
</cfg>
".Replace("'", "\"");
var root = new TflRoot(xml);
var problems = root.Errors();
foreach (var problem in problems)
{
Console.WriteLine(problem);
}
Assert.AreEqual(0, problems.Length);
Assert.AreEqual(3, root.Processes.First().DataSets.First().Rows.Count);
var rows = ProcessFactory.CreateSingle(root.Processes[0], new TestLogger()).Execute().ToList();
Assert.AreEqual(3, rows.Count());
}
public void TestDefaultParameters()
{
const string xml = @"<transformalize>
<environments default='@(Environment)'>
<add name='one'>
<parameters>
<add name=""t1"" value=""v1"" />
<add name=""t2"" value=""v2"" />
</parameters>
</add>
<add name='two'>
<parameters>
<add name=""t1"" value=""v2"" />
<add name=""t2"" value=""v3"" />
</parameters>
</add>
</environments>
<processes>
<add name=""test1"">
<actions>
<add action=""@(t1)"" />
</actions>
</add>
<add name=""test2"">
<actions>
<add action=""@(t1)"" />
<add action=""@(t2)"" />
</actions>
</add>
</processes>
</transformalize>";
var root1 = new TflRoot(xml, new Dictionary <string, string>()
{
{ "Environment", "one" }
});
Assert.AreEqual("v1", root1.Processes[0].Actions[0].Action);
Assert.AreEqual("v1", root1.Processes[1].Actions[0].Action);
Assert.AreEqual("v2", root1.Processes[1].Actions[1].Action);
var root2 = new TflRoot(xml, new Dictionary <string, string>()
{
{ "Environment", "two" }
});
Assert.AreEqual("v2", root2.Processes[0].Actions[0].Action);
Assert.AreEqual("v2", root2.Processes[1].Actions[0].Action);
Assert.AreEqual("v3", root2.Processes[1].Actions[1].Action);
}
public TransformalizeRequest(
ConfigurationPart part,
Dictionary <string, string> query,
string modifiedConfiguration,
ILogger logger,
TflRoot root = null)
{
Part = part;
Configuration = modifiedConfiguration ?? part.Configuration;
Query = query ?? new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
Options = new Options {
Mode = Query["mode"]
};
Root = root ?? new TflRoot(modifiedConfiguration ?? part.Configuration, Query, new CfgNetLogger(logger));
}
public ShortHandFactory GetFactory(TflField field = null)
{
var process = new TflRoot().GetDefaultOf <TflProcess>(p => {
p.Name = "Test";
p.Entities = new List <TflEntity> {
new TflEntity().GetDefaultOf <TflEntity>(e => { e.Name = "Test"; })
};
});
if (field != null)
{
process.Entities.First().Fields.Add(field);
}
return(new ShortHandFactory(process));
}
protected override void Updated(UpdateContentContext context)
{
var part = context.ContentItem.As <ConfigurationPart>();
if (part == null)
{
return;
}
try {
//test if configuration works
var root = new TflRoot(
part.Configuration,
null,
new CfgNetNotifier(_notifier)
);
CheckAddress(part.StartAddress);
CheckAddress(part.EndAddress);
Logger.Information("Loaded {0} with {1} warnings, and {2} errors.", part.Title(), root.Warnings().Length, root.Errors().Length);
} catch (Exception ex) {
_notifier.Add(NotifyType.Warning, T(ex.Message));
Logger.Warning(ex.Message);
}
}
private static TflProcess BuildProcess(
FileInformation fileInformation,
FileInspectionRequest request,
TflConnection output,
ILogger logger)
{
var process = new TflRoot().GetDefaultOf <TflProcess>(p => {
p.Name = request.EntityName;
p.Star = request.ProcessName;
p.StarEnabled = false;
p.ViewEnabled = false;
p.PipelineThreading = "MultiThreaded";
p.Connections = new List <TflConnection> {
p.GetDefaultOf <TflConnection>(c => {
c.Name = "input";
c.Provider = "file";
c.File = fileInformation.FileInfo.FullName;
c.Delimiter = fileInformation.Delimiter == default(char)
? "|"
: fileInformation.Delimiter.ToString(CultureInfo.InvariantCulture);
c.Start = fileInformation.FirstRowIsHeader ? 2 : 1;
}),
output
};
p.Entities = new List <TflEntity> {
p.GetDefaultOf <TflEntity>(e => {
e.Name = request.EntityName;
e.Connection = "input";
e.PrependProcessNameToOutputName = false;
e.DetectChanges = false;
e.Fields = GetFields(p, new FieldInspector(logger).Inspect(fileInformation, request), logger, request.EntityName);
})
};
});
return(process);
}
public Fields Inspect(FileInformation fileInformation, FileInspectionRequest request)
{
var process = new TflRoot().GetDefaultOf <TflProcess>(p => {
p.Name = request.ProcessName;
p.StarEnabled = false;
p.ViewEnabled = false;
p.PipelineThreading = "MultiThreaded";
});
process.Connections = new List <TflConnection> {
process.GetDefaultOf <TflConnection>(c => {
c.Name = "input";
c.Provider = "file";
c.File = fileInformation.FileInfo.FullName;
c.Delimiter = fileInformation.Delimiter == default(char)
? "|"
: fileInformation.Delimiter.ToString(CultureInfo.InvariantCulture);
c.Start = fileInformation.FirstRowIsHeader ? 2 : 1;
}),
process.GetDefaultOf <TflConnection>(c => {
c.Name = "output";
c.Provider = "internal";
})
};
process.Entities.Add(process.GetDefaultOf <TflEntity>(e => {
e.Name = request.EntityName;
e.PrependProcessNameToOutputName = false;
e.DetectChanges = false;
e.Sample = System.Convert.ToInt32(request.Sample);
}));
foreach (var fd in fileInformation.Fields)
{
var field = fd;
process.Entities[0].Fields.Add(process.GetDefaultOf <TflField>(f => {
f.Name = field.Name;
f.Length = field.Length;
f.Type = field.Type;
f.QuotedWith = field.QuotedWith;
}));
}
for (var i = 0; i < request.DataTypes.Count; i++)
{
var dataType = request.DataTypes[i];
foreach (var field in fileInformation.Fields)
{
var result = IsDataTypeField(field.Name, dataType);
process.Entities[0].CalculatedFields.Add(
process.GetDefaultOf <TflField>(f => {
f.Name = result;
f.Type = "bool";
f.Input = false;
f.Transforms = new List <TflTransform> {
f.GetDefaultOf <TflTransform>(t => {
t.Method = "typeconversion";
t.Type = dataType;
t.Parameter = field.Name;
t.IgnoreEmpty = request.IgnoreEmpty;
})
};
})
);
}
}
foreach (var field in fileInformation.Fields)
{
var result = LengthField(field.Name);
process.Entities[0].CalculatedFields.Add(
process.GetDefaultOf <TflField>(f => {
f.Name = result;
f.Type = "int32";
f.Transforms = new List <TflTransform> {
f.GetDefaultOf <TflTransform>(t => {
t.Method = "length";
t.Parameter = field.Name;
})
};
})
);
}
var runner = ProcessFactory.CreateSingle(new TflRoot(process).Processes[0], _logger);
var results = runner.Execute().ToList();
if (results.Count <= 0)
{
_logger.Warn("Nothing imported from in {0}!", fileInformation.FileInfo.Name);
return(fileInformation.Fields);
}
foreach (var field in fileInformation.Fields)
{
if (!results.All(row => row[field.Name].Equals(string.Empty)))
{
foreach (var dataType in request.DataTypes)
{
var result = IsDataTypeField(field.Name, dataType);
if (!results.All(row => row[result].Equals(true)))
{
continue;
}
field.Type = dataType;
field.Length = request.MinLength.ToString(CultureInfo.InvariantCulture);
break;
}
}
if (!field.Type.Equals("string"))
{
continue;
}
var length = results.Max(row => (int)row[LengthField(field.Name)]) + 1;
if (request.MaxLength > 0 && length > request.MaxLength)
{
length = request.MaxLength;
}
if (request.MinLength > 0 && length < request.MinLength)
{
length = request.MinLength;
}
field.Length = length.ToString(CultureInfo.InvariantCulture);
}
return(fileInformation.Fields);
}
private static Process GetTestProcess(string outputProvider)
{
var process = new ProcessBuilder("Test")
.StarEnabled(false)
.Connection("input").Provider("internal")
.Connection("output").Provider(outputProvider)
.Entity("OrderDetail")
.Schema("dbo")
.Field("OrderDetailId").PrimaryKey()
.Field("OrderId")
.Field("ProductId")
.Field("Price")
.Field("Quantity")
.Entity("Order")
.Schema("dbo")
.Field("OrderId").PrimaryKey()
.Field("CustomerId")
.Entity("Customer")
.Schema("dbo")
.Field("CustomerId").PrimaryKey()
.Field("FirstName")
.Field("LastName")
.Entity("Product")
.Schema("dbo")
.Field("ProductId").PrimaryKey()
.Field("Name").Alias("Product")
.Field("Description")
.Entity("ProductCategory")
.Schema("dbo")
.Field("ProductId").PrimaryKey()
.Field("CategoryId").PrimaryKey()
.Entity("Category")
.Schema("dbo")
.Field("CategoryId").PrimaryKey()
.Field("Name").Alias("Category")
.Relationship()
.LeftEntity("OrderDetail").LeftField("OrderId")
.RightEntity("Order").RightField("OrderId")
.Relationship()
.LeftEntity("Order").LeftField("CustomerId")
.RightEntity("Customer").RightField("CustomerId")
.Relationship()
.LeftEntity("OrderDetail").LeftField("ProductId")
.RightEntity("Product").RightField("ProductId")
.Relationship()
.LeftEntity("Product").LeftField("ProductId")
.RightEntity("ProductCategory").RightField("ProductId")
.Relationship()
.LeftEntity("ProductCategory").LeftField("CategoryId")
.RightEntity("Category").RightField("CategoryId")
.Process();
process = new TflRoot(process).Processes[0];
process.Entities[0].InputOperation = new RowsBuilder()
.Row("OrderDetailId", 1).Field("OrderId", 1).Field("ProductId", 1).Field("Quantity", 2).Field("Price", 2.0)
.Row("OrderDetailId", 2).Field("OrderId", 1).Field("ProductId", 2).Field("Quantity", 2).Field("Price", 3.0)
.Row("OrderDetailId", 3).Field("OrderId", 2).Field("ProductId", 2).Field("Quantity", 4).Field("Price", 3.0)
.ToOperation();
process.Entities[1].InputOperation = new RowsBuilder()
.Row("OrderId", 1).Field("CustomerId", 1)
.Row("OrderId", 2).Field("CustomerId", 2)
.ToOperation();
process.Entities[2].InputOperation = new RowsBuilder()
.Row("CustomerId", 1).Field("FirstName", "Dale").Field("LastName", "Newman")
.Row("CustomerId", 2).Field("FirstName", "Tara").Field("LastName", "Newman")
.ToOperation();
process.Entities[3].InputOperation = new RowsBuilder()
.Row("ProductId", 1).Field("Name", "Beans").Field("Description", "Really nice beans.")
.Row("ProductId", 2).Field("Name", "Cheese").Field("Description", "Very sharp cheese.")
.ToOperation();
process.Entities[4].InputOperation = new RowsBuilder()
.Row("ProductId", 1).Field("CategoryId", 1)
.Row("ProductId", 1).Field("CategoryId", 2)
.Row("ProductId", 2).Field("CategoryId", 3)
.ToOperation();
process.Entities[5].InputOperation = new RowsBuilder()
.Row("CategoryId", 1).Field("Name", "Proteins")
.Row("CategoryId", 2).Field("Name", "Miracle Fruits")
.Row("CategoryId", 3).Field("Name", "Dairy")
.ToOperation();
return(ProcessFactory.CreateSingle(process, new TestLogger()));
}
/// <summary>
/// Create process(es) from a configuration element.
/// </summary>
/// <param name="element">The Cfg-NET configuration element.</param>
/// <param name="logger">An ILogger implementation.</param>
/// <param name="options"></param>
/// <returns></returns>
public static Process[] Create(TflRoot element, ILogger logger, Options options = null)
{
return(Create(element.Processes, logger, options));
}
public static string GetContent(ApiRequest request, string configuration, TransformalizeResponse response)
{
var builder = new StringBuilder();
var processes = "[]";
var environments = "[]";
var results = "[]";
const string content = "{}";
var tfl = new TflRoot(configuration);
var settings = new JsonSerializerSettings {
ContractResolver = new CamelCasePropertyNamesContractResolver()
};
switch (request.RequestType)
{
case ApiRequestType.MetaData:
processes = JsonConvert.SerializeObject(tfl.Processes, Formatting.None, settings);
builder.AppendFormat(JSON_TEMPLATE, request.RequestType, request.Status, request.Message, request.Stopwatch.ElapsedMilliseconds, environments, processes, results, LogsToJson(response.Log), content);
return(builder.ToString());
case ApiRequestType.Configuration:
processes = JsonConvert.SerializeObject(tfl.Processes, Formatting.None, settings);
environments = JsonConvert.SerializeObject(tfl.Environments, Formatting.None, settings);
builder.AppendFormat(JSON_TEMPLATE, request.RequestType, request.Status, request.Message, request.Stopwatch.ElapsedMilliseconds, environments, processes, results, LogsToJson(response.Log), content);
return(builder.ToString());
case ApiRequestType.Enqueue:
builder.AppendFormat(JSON_TEMPLATE, request.RequestType, request.Status, request.Message, request.Stopwatch.ElapsedMilliseconds, environments, processes, results, LogsToJson(response.Log), content);
return(builder.ToString());
case ApiRequestType.Execute:
foreach (var process in tfl.Processes)
{
process.Connections = new List <TflConnection>();
}
processes = JsonConvert.SerializeObject(tfl.Processes, Formatting.None, settings);
switch (request.Flavor)
{
case "arrays":
results = new JsonResultsToArrayHandler().Handle(response.Processes);
break;
case "array":
goto case "arrays";
case "dictionaries":
results = new JsonResultsToDictionaryHandler().Handle(response.Processes);
break;
case "dictionary":
goto case "dictionaries";
default:
results = new JsonResultsToObjectHandler().Handle(response.Processes);
break;
}
builder.AppendFormat(JSON_TEMPLATE, request.RequestType, request.Status, request.Message, request.Stopwatch.ElapsedMilliseconds, environments, processes, results, LogsToJson(response.Log), content);
return(builder.ToString());
default:
if (request.Status == 200)
{
request.Status = 400;
request.Message = "Bad Request";
}
builder.AppendFormat(JSON_TEMPLATE, request.RequestType, request.Status, request.Message, request.Stopwatch.ElapsedMilliseconds, environments, processes, results, LogsToJson(response.Log), content);
return(builder.ToString());
}
}
protected static void TransformConfigurationForLoad(TflRoot root)
{
ReverseConfiguration(root);
}
