public List<TflProcess> Create() {
var shouldThrow = false;
var cfg = new TflRoot(_resource, _parameters);
if (cfg.Response.Any()) {
foreach (var response in cfg.Response.Where(response => response.Status != (short)200)) {
_logger.Warn("API at {0} responded with {1} {2}.", _resource, response.Status, response.Message);
}
}
foreach (var warning in cfg.Warnings()) {
_logger.Warn(warning);
}
foreach (var error in cfg.Errors()) {
shouldThrow = true;
_logger.Error(error);
}
if (shouldThrow) {
throw new TransformalizeException(_logger, string.Join(Environment.NewLine, cfg.Errors()));
}
return cfg.Processes;
}
public ActionResult Load(int id) {
Response.AddHeader("Access-Control-Allow-Origin", "*");
ConfigurationPart part;
ApiRequest request;
foreach (var rejection in _apiService.Rejections(id, out request, out part)) {
return rejection.ContentResult(_query["format"], _query["flavor"]);
}
request.RequestType = ApiRequestType.Execute;
var root = new TflRoot(part.Configuration, _query);
TransformConfigurationForLoad(root);
var errors = root.Errors();
if (errors.Any()) {
request.Status = 500;
request.Message = string.Join(Environment.NewLine, errors);
return new ApiResponse(request, root.ToString(), new TransformalizeResponse()).ContentResult(_query["format"], _query["flavor"]);
}
var transformalizeRequest = new TransformalizeRequest(part, _query, null, Logger, root);
return Run(request, transformalizeRequest)
.ContentResult(_query["format"], _query["flavor"]);
}
protected static void TransformConfigurationForDownload(TflRoot root, Dictionary<string, string> query) {
ReverseConfiguration(root);
var output = root.Processes[0].Connections.First(c => c.Name == "output");
output.Provider = "file";
output.File = query.ContainsKey("OutputFile") ? query["OutputFile"] : string.Empty;
}
public TransformalizeRequest(
ConfigurationPart part,
Dictionary<string, string> query,
string modifiedConfiguration,
ILogger logger,
TflRoot root = null) {
Part = part;
Configuration = modifiedConfiguration ?? part.Configuration;
Query = query ?? new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
Options = new Options { Mode = Query["mode"] };
Root = root ?? new TflRoot(modifiedConfiguration ?? part.Configuration, Query, new CfgNetLogger(logger));
}
public ShortHandFactory GetFactory(TflField field = null) {
var process = new TflRoot().GetDefaultOf<TflProcess>(p => {
p.Name = "Test";
p.Entities = new List<TflEntity>{
new TflEntity().GetDefaultOf<TflEntity>(e => { e.Name = "Test"; })
};
});
if (field != null) {
process.Entities.First().Fields.Add(field);
}
return new ShortHandFactory(process);
}
public void Test2() {
var xml = @"
<cfg>
<processes>
<add name='test'>
<data-sets>
<add name='one'>
<rows>
<add f1='1' f2='1' f3='2001-01-01' />
<add f1='2' f2='2' f3='2002-02-02' />
<add f1='3' f2='3' f3='2003-03-03' />
</rows>
</add>
</data-sets>
<connections>
<add name='input' provider='internal' />
<add name='output' provider='internal' />
</connections>
<entities>
<add name='one'>
<fields>
<add name='f1' primary-key='true' />
<add name='f2' type='int' />
<add name='f3' type='datetime' />
</fields>
</add>
</entities>
</add>
</processes>
</cfg>
".Replace("'","\"");
var root = new TflRoot(xml);
var problems = root.Errors();
foreach (var problem in problems) {
Console.WriteLine(problem);
}
Assert.AreEqual(0, problems.Length);
Assert.AreEqual(3, root.Processes.First().DataSets.First().Rows.Count);
var rows = ProcessFactory.CreateSingle(root.Processes[0], new TestLogger()).Execute().ToList();
Assert.AreEqual(3, rows.Count());
}
// Custom constructor takeing an already created TflProcess
public TflRoot(TflProcess process)
{
var root = new TflRoot {
Processes = new List <TflProcess> {
process
}
};
var json = JsonConvert.SerializeObject(root, new JsonSerializerSettings()
{
NullValueHandling = NullValueHandling.Ignore
});
Load(json);
}
public TestWithProcess() {
var cfg = new TflRoot(File.ReadAllText(@"TestFiles\Test.xml"), null);
foreach (var problem in cfg.Errors()) {
Console.WriteLine(problem);
}
_process = ProcessFactory.CreateSingle(cfg.Processes[0], new TestLogger());
_entityKeysExtract = new Mock<IOperation>();
_entityKeysExtract.Setup(foo => foo.Execute(It.IsAny<IEnumerable<Row>>())).Returns(new List<Row> {
new Row { {"OrderDetailKey", 1} },
new Row { {"OrderDetailKey", 2} },
new Row { {"OrderDetailKey", 3} },
new Row { {"OrderDetailKey", 4} }
});
}
protected static void ReverseConfiguration(TflRoot root) {
var process = root.Processes[0];
process.StarEnabled = false;
//swap input and output
process.Connections.First(c => c.Name.Equals("output")).Name = "temp";
process.Connections.First(c => c.Name.Equals("input")).Name = "output";
process.Connections.First(c => c.Name.Equals("temp")).Name = "input";
var entity = process.Entities[0];
entity.Name = GetEntityOutputName(entity, process.Name);
//if no primary key is set, make all the fields a composite primary key
if (!entity.Fields.Any(f => f.PrimaryKey)) {
foreach (var field in entity.Fields) {
field.PrimaryKey = true;
}
}
//aliases are used to write to output, so swap when reversing
foreach (var field in entity.Fields) {
var temp = field.Name;
field.Name = field.Alias;
field.Alias = temp;
}
entity.DetectChanges = false;
if (!entity.Delete)
return;
entity.Delete = false;
entity.Filter.Add(new TflFilter {
Left = "TflDeleted",
Right = "0"
}.WithDefaults());
entity.Fields.Add(new TflField {
Name = "TflDeleted",
Type = "bool",
Output = false,
Label = "Deleted"
}.WithDefaults());
}
protected override void Updated(UpdateContentContext context) {
var part = context.ContentItem.As<ConfigurationPart>();
if (part == null)
return;
try {
//test if configuration works
var root = new TflRoot(
part.Configuration,
null,
new CfgNetNotifier(_notifier)
);
CheckAddress(part.StartAddress);
CheckAddress(part.EndAddress);
Logger.Information("Loaded {0} with {1} warnings, and {2} errors.", part.Title(), root.Warnings().Length, root.Errors().Length);
} catch (Exception ex) {
_notifier.Add(NotifyType.Warning, T(ex.Message));
Logger.Warning(ex.Message);
}
}
public void TestDefaultParameters() {
const string xml = @"<transformalize environment='@(Environment)'>
<environments>
<add name='one'>
<parameters>
<add name=""t1"" value=""v1"" />
<add name=""t2"" value=""v2"" />
</parameters>
</add>
<add name='two'>
<parameters>
<add name=""t1"" value=""v2"" />
<add name=""t2"" value=""v3"" />
</parameters>
</add>
</environments>
<processes>
<add name=""test1"">
<actions>
<add action=""@(t1)"" />
</actions>
</add>
<add name=""test2"">
<actions>
<add action=""@(t1)"" />
<add action=""@(t2)"" />
</actions>
</add>
</processes>
</transformalize>";
var root1 = new TflRoot(xml, new Dictionary<string, string>() { { "Environment", "one" } });
Assert.AreEqual("v1", root1.Processes[0].Actions[0].Action);
Assert.AreEqual("v1", root1.Processes[1].Actions[0].Action);
Assert.AreEqual("v2", root1.Processes[1].Actions[1].Action);
var root2 = new TflRoot(xml, new Dictionary<string, string>() { { "Environment", "two" } });
Assert.AreEqual("v2", root2.Processes[0].Actions[0].Action);
Assert.AreEqual("v2", root2.Processes[1].Actions[0].Action);
Assert.AreEqual("v3", root2.Processes[1].Actions[1].Action);
}
public void TestError() {
var cfg = new TflRoot(File.ReadAllText(@"C:\Temp\test.xml"), null);
Assert.AreEqual(0, cfg.Errors().Length);
var process = ProcessFactory.CreateSingle(@"C:\Temp\test.xml", new TestLogger());
Assert.IsNotNull(process);
process.ExecuteScaler();
}
public Fields Inspect(FileInformation fileInformation, FileInspectionRequest request) {
var process = new TflRoot().GetDefaultOf<TflProcess>(p => {
p.Name = request.ProcessName;
p.StarEnabled = false;
p.ViewEnabled = false;
p.PipelineThreading = "MultiThreaded";
});
process.Connections = new List<TflConnection> {
process.GetDefaultOf<TflConnection>(c => {
c.Name = "input";
c.Provider = "file";
c.File = fileInformation.FileInfo.FullName;
c.Delimiter = fileInformation.Delimiter == default(char)
? "|"
: fileInformation.Delimiter.ToString(CultureInfo.InvariantCulture);
c.Start = fileInformation.FirstRowIsHeader ? 2 : 1;
}),
process.GetDefaultOf<TflConnection>(c => {
c.Name = "output";
c.Provider = "internal";
})
};
process.Entities.Add(process.GetDefaultOf<TflEntity>(e => {
e.Name = request.EntityName;
e.PrependProcessNameToOutputName = false;
e.DetectChanges = false;
e.Sample = System.Convert.ToInt32(request.Sample);
}));
foreach (var fd in fileInformation.Fields) {
var field = fd;
process.Entities[0].Fields.Add(process.GetDefaultOf<TflField>(f => {
f.Name = field.Name;
f.Length = field.Length;
f.Type = field.Type;
f.QuotedWith = field.QuotedWith;
}));
}
for (var i = 0; i < request.DataTypes.Count; i++) {
var dataType = request.DataTypes[i];
foreach (var field in fileInformation.Fields) {
var result = IsDataTypeField(field.Name, dataType);
process.Entities[0].CalculatedFields.Add(
process.GetDefaultOf<TflField>(f => {
f.Name = result;
f.Type = "bool";
f.Input = false;
f.Transforms = new List<TflTransform> {
f.GetDefaultOf<TflTransform>(t => {
t.Method = "typeconversion";
t.Type = dataType;
t.Parameter = field.Name;
t.IgnoreEmpty = request.IgnoreEmpty;
})
};
})
);
}
}
foreach (var field in fileInformation.Fields) {
var result = LengthField(field.Name);
process.Entities[0].CalculatedFields.Add(
process.GetDefaultOf<TflField>(f => {
f.Name = result;
f.Type = "int32";
f.Transforms = new List<TflTransform> {
f.GetDefaultOf<TflTransform>(t => {
t.Method = "length";
t.Parameter = field.Name;
})
};
})
);
}
var runner = ProcessFactory.CreateSingle(new TflRoot(process).Processes[0], _logger);
var results = runner.Execute().ToList();
if (results.Count <= 0) {
_logger.Warn("Nothing imported from in {0}!", fileInformation.FileInfo.Name);
return fileInformation.Fields;
}
foreach (var field in fileInformation.Fields) {
if (!results.All(row => row[field.Name].Equals(string.Empty))) {
foreach (var dataType in request.DataTypes) {
var result = IsDataTypeField(field.Name, dataType);
if (!results.All(row => row[result].Equals(true)))
continue;
field.Type = dataType;
field.Length = request.MinLength.ToString(CultureInfo.InvariantCulture);
break;
}
}
if (!field.Type.Equals("string"))
continue;
var length = results.Max(row => (int)row[LengthField(field.Name)]) + 1;
if (request.MaxLength > 0 && length > request.MaxLength) {
length = request.MaxLength;
}
if (request.MinLength > 0 && length < request.MinLength) {
length = request.MinLength;
}
field.Length = length.ToString(CultureInfo.InvariantCulture);
}
return fileInformation.Fields;
}
protected static void TransformConfigurationForLoad(TflRoot root) {
ReverseConfiguration(root);
}
// Custom constructor takeing an already created TflProcess
public TflRoot(TflProcess process) {
var root = new TflRoot { Processes = new List<TflProcess> { process } };
var json = JsonConvert.SerializeObject(root, new JsonSerializerSettings() {NullValueHandling = NullValueHandling.Ignore});
Load(json);
}
private static TflProcess BuildProcess(
FileInformation fileInformation,
FileInspectionRequest request,
TflConnection output,
ILogger logger) {
var process = new TflRoot().GetDefaultOf<TflProcess>(p => {
p.Name = request.EntityName;
p.Star = request.ProcessName;
p.StarEnabled = false;
p.ViewEnabled = false;
p.PipelineThreading = "MultiThreaded";
p.Connections = new List<TflConnection> {
p.GetDefaultOf<TflConnection>(c => {
c.Name = "input";
c.Provider = "file";
c.File = fileInformation.FileInfo.FullName;
c.Delimiter = fileInformation.Delimiter == default(char)
? "|"
: fileInformation.Delimiter.ToString(CultureInfo.InvariantCulture);
c.Start = fileInformation.FirstRowIsHeader ? 2 : 1;
}),
output
};
p.Entities = new List<TflEntity> {
p.GetDefaultOf<TflEntity>(e => {
e.Name = request.EntityName;
e.Connection = "input";
e.PrependProcessNameToOutputName = false;
e.DetectChanges = false;
e.Fields = GetFields(p, new FieldInspector(logger).Inspect(fileInformation, request), logger, request.EntityName);
})
};
});
return process;
}
/// <summary>
/// Create process(es) from a configuration element.
/// </summary>
/// <param name="element">The Cfg-NET configuration element.</param>
/// <param name="logger">An ILogger implementation.</param>
/// <param name="options"></param>
/// <returns></returns>
public static Process[] Create(TflRoot element, ILogger logger, Options options = null) {
return Create(element.Processes, logger, options);
}
private static Process GetTestProcess(string outputProvider) {
var process = new ProcessBuilder("Test")
.StarEnabled(false)
.Connection("input").Provider("internal")
.Connection("output").Provider(outputProvider)
.Entity("OrderDetail")
.Schema("dbo")
.Field("OrderDetailId").PrimaryKey()
.Field("OrderId")
.Field("ProductId")
.Field("Price")
.Field("Quantity")
.Entity("Order")
.Schema("dbo")
.Field("OrderId").PrimaryKey()
.Field("CustomerId")
.Entity("Customer")
.Schema("dbo")
.Field("CustomerId").PrimaryKey()
.Field("FirstName")
.Field("LastName")
.Entity("Product")
.Schema("dbo")
.Field("ProductId").PrimaryKey()
.Field("Name").Alias("Product")
.Field("Description")
.Entity("ProductCategory")
.Schema("dbo")
.Field("ProductId").PrimaryKey()
.Field("CategoryId").PrimaryKey()
.Entity("Category")
.Schema("dbo")
.Field("CategoryId").PrimaryKey()
.Field("Name").Alias("Category")
.Relationship()
.LeftEntity("OrderDetail").LeftField("OrderId")
.RightEntity("Order").RightField("OrderId")
.Relationship()
.LeftEntity("Order").LeftField("CustomerId")
.RightEntity("Customer").RightField("CustomerId")
.Relationship()
.LeftEntity("OrderDetail").LeftField("ProductId")
.RightEntity("Product").RightField("ProductId")
.Relationship()
.LeftEntity("Product").LeftField("ProductId")
.RightEntity("ProductCategory").RightField("ProductId")
.Relationship()
.LeftEntity("ProductCategory").LeftField("CategoryId")
.RightEntity("Category").RightField("CategoryId")
.Process();
process = new TflRoot(process).Processes[0];
process.Entities[0].InputOperation = new RowsBuilder()
.Row("OrderDetailId", 1).Field("OrderId", 1).Field("ProductId", 1).Field("Quantity", 2).Field("Price", 2.0)
.Row("OrderDetailId", 2).Field("OrderId", 1).Field("ProductId", 2).Field("Quantity", 2).Field("Price", 3.0)
.Row("OrderDetailId", 3).Field("OrderId", 2).Field("ProductId", 2).Field("Quantity", 4).Field("Price", 3.0)
.ToOperation();
process.Entities[1].InputOperation = new RowsBuilder()
.Row("OrderId", 1).Field("CustomerId", 1)
.Row("OrderId", 2).Field("CustomerId", 2)
.ToOperation();
process.Entities[2].InputOperation = new RowsBuilder()
.Row("CustomerId", 1).Field("FirstName", "Dale").Field("LastName", "Newman")
.Row("CustomerId", 2).Field("FirstName", "Tara").Field("LastName", "Newman")
.ToOperation();
process.Entities[3].InputOperation = new RowsBuilder()
.Row("ProductId", 1).Field("Name", "Beans").Field("Description", "Really nice beans.")
.Row("ProductId", 2).Field("Name", "Cheese").Field("Description", "Very sharp cheese.")
.ToOperation();
process.Entities[4].InputOperation = new RowsBuilder()
.Row("ProductId", 1).Field("CategoryId", 1)
.Row("ProductId", 1).Field("CategoryId", 2)
.Row("ProductId", 2).Field("CategoryId", 3)
.ToOperation();
process.Entities[5].InputOperation = new RowsBuilder()
.Row("CategoryId", 1).Field("Name", "Proteins")
.Row("CategoryId", 2).Field("Name", "Miracle Fruits")
.Row("CategoryId", 3).Field("Name", "Dairy")
.ToOperation();
return ProcessFactory.CreateSingle(process, new TestLogger());
}
public void TestEnvironmentParameters() {
const string xml = @"<transformalize>
<environments>
<add name=""e1"">
<parameters>
<add name=""t1"" value=""v1"" />
<add name=""t2"" value=""v2"" />
<add name=""t3"" value=""v3"" />
</parameters>
</add>
</environments>
<processes>
<add name=""test1"">
<actions>
<add action=""@(t1)"" />
</actions>
</add>
<add name=""test2"">
<actions>
<add action=""@(t2)"" />
<add action=""@(t3)"" />
</actions>
</add>
</processes>
</transformalize>";
var processes = new TflRoot(xml, null).Processes;
Assert.AreEqual("v1", processes[0].Actions[0].Action);
Assert.AreEqual("v2", processes[1].Actions[0].Action);
Assert.AreEqual("v3", processes[1].Actions[1].Action);
}
public static string GetContent(ApiRequest request, string configuration, TransformalizeResponse response) {
var builder = new StringBuilder();
var processes = "[]";
var environments = "[]";
var results = "[]";
const string content = "{}";
var tfl = new TflRoot(configuration);
var settings = new JsonSerializerSettings {
ContractResolver = new CamelCasePropertyNamesContractResolver()
};
switch (request.RequestType) {
case ApiRequestType.MetaData:
processes = JsonConvert.SerializeObject(tfl.Processes, Formatting.None, settings);
builder.AppendFormat(JSON_TEMPLATE, request.RequestType, request.Status, request.Message, request.Stopwatch.ElapsedMilliseconds, environments, processes, results, LogsToJson(response.Log), content);
return builder.ToString();
case ApiRequestType.Configuration:
processes = JsonConvert.SerializeObject(tfl.Processes, Formatting.None, settings);
environments = JsonConvert.SerializeObject(tfl.Environments, Formatting.None, settings);
builder.AppendFormat(JSON_TEMPLATE, request.RequestType, request.Status, request.Message, request.Stopwatch.ElapsedMilliseconds, environments, processes, results, LogsToJson(response.Log), content);
return builder.ToString();
case ApiRequestType.Enqueue:
builder.AppendFormat(JSON_TEMPLATE, request.RequestType, request.Status, request.Message, request.Stopwatch.ElapsedMilliseconds, environments, processes, results, LogsToJson(response.Log), content);
return builder.ToString();
case ApiRequestType.Execute:
foreach (var process in tfl.Processes) {
process.Connections = new List<TflConnection>();
}
processes = JsonConvert.SerializeObject(tfl.Processes, Formatting.None, settings);
switch (request.Flavor) {
case "arrays":
results = new JsonResultsToArrayHandler().Handle(response.Processes);
break;
case "array":
goto case "arrays";
case "dictionaries":
results = new JsonResultsToDictionaryHandler().Handle(response.Processes);
break;
case "dictionary":
goto case "dictionaries";
default:
results = new JsonResultsToObjectHandler().Handle(response.Processes);
break;
}
builder.AppendFormat(JSON_TEMPLATE, request.RequestType, request.Status, request.Message, request.Stopwatch.ElapsedMilliseconds, environments, processes, results, LogsToJson(response.Log), content);
return builder.ToString();
default:
if (request.Status == 200) {
request.Status = 400;
request.Message = "Bad Request";
}
builder.AppendFormat(JSON_TEMPLATE, request.RequestType, request.Status, request.Message, request.Stopwatch.ElapsedMilliseconds, environments, processes, results, LogsToJson(response.Log), content);
return builder.ToString();
}
}