public IProcessController Compose(string cfg, LogLevel logLevel = LogLevel.Info, Dictionary <string, string> parameters = null, string placeHolderStyle = "@()")
{
var logger = new TraceLogger(logLevel);
var container = ConfigurationContainer.Create(cfg, logger, parameters, placeHolderStyle);
Process = parameters == null?container.Resolve <Process>(new NamedParameter("cfg", cfg)) : container.Resolve <Process>(new NamedParameter("cfg", cfg), new NamedParameter("parameters", parameters));
if (Process.Errors().Any())
{
foreach (var error in Process.Errors())
{
Trace.WriteLine(error);
}
throw new Exception("Configuration Error(s)");
}
if (Process.Warnings().Any())
{
foreach (var warning in Process.Warnings())
{
Trace.WriteLine(warning);
}
}
return(DefaultContainer.Create(Process, logger, placeHolderStyle).Resolve <IProcessController>(new NamedParameter("cfg", cfg)));
}
public void Execute(Process process)
{
var logger = _logger ?? new NLogPipelineLogger(SlugifyTransform.Slugify(Cfg));
if (process.OutputIsConsole())
{
logger.SuppressConsole();
}
if (RunCount % 3 == 0)
{
if (_checkMemory)
{
CheckMemory(process, logger);
}
else
{
var context = new PipelineContext(logger, process);
context.Info($"Process has run {RunCount} time{RunCount.Plural()}.");
}
}
using (var scope = DefaultContainer.Create(process, logger, Options.PlaceHolderStyle)) {
try {
scope.Resolve <IProcessController>().Execute();
} catch (Exception ex) {
var context = scope.Resolve <IContext>();
context.Error(ex.Message);
context.Logger.Clear();
new LibaryVersionChecker(context).Check();
}
}
++RunCount;
}
public IProcessController Compose(string cfg, LogLevel logLevel = LogLevel.Info, Dictionary <string, string> parameters = null)
{
var builder = new ContainerBuilder();
builder.RegisterModule(new RootModule(@"Shorthand.xml"));
var container = builder.Build();
Process = parameters == null?container.Resolve <Process>(new NamedParameter("cfg", cfg)) : container.Resolve <Process>(new NamedParameter("cfg", cfg), new NamedParameter("parameters", parameters));
if (Process.Errors().Any())
{
foreach (var error in Process.Errors())
{
Trace.WriteLine(error);
}
throw new Exception("Configuration Error(s)");
}
if (Process.Warnings().Any())
{
foreach (var warning in Process.Warnings())
{
Trace.WriteLine(warning);
}
}
return(DefaultContainer.Create(Process, new TraceLogger(logLevel)).Resolve <IProcessController>(new NamedParameter("cfg", cfg)));
}
public void Execute(Process process)
{
var logger = _logger ?? new NLogPipelineLogger(SlugifyTransform.Slugify(Cfg));
if (process.OutputIsConsole())
{
logger.SuppressConsole();
}
if (_checkMemory)
{
CheckMemory(process, logger);
}
using (var scope = DefaultContainer.Create(process, logger, Options.PlaceHolderStyle)) {
try {
scope.Resolve <IProcessController>().Execute();
} catch (Exception ex) {
var context = scope.Resolve <IContext>();
context.Error(ex.Message);
context.Logger.Clear();
new LibaryVersionChecker(context).Check();
}
}
}
public void A()
{
const string xml = @"
<add name='TestProcess'>
<parameters>
<add name='humanized' value='camelCase' t='replace(lC,l-c).upper()' />
<add name='replaced' value='TestEnvironment' t='replace(Test,Production).trimEnd(s).append(s)' />
</parameters>
<entities>
</entities>
</add>";
var logger = new TraceLogger(LogLevel.Debug);
using (var outer = ConfigurationContainer.Create(xml, logger)) {
var process = outer.Resolve <Process>();
using (var inner = DefaultContainer.Create(process, logger, "@()")) {
inner.Resolve <IProcessController>().Execute();
var parmeters = process.GetActiveParameters();
Assert.AreEqual("CAMEL-CASE", parmeters.First().Value);
Assert.AreEqual("ProductionEnvironments", parmeters.Skip(1).First().Value);
}
}
}
public Schema Read()
{
if (Process == null)
{
_host.Error("RunTimeSchemaReader executed without a Process");
return(new Schema());
}
using (var scope = DefaultContainer.Create(Process, _host.Logger, _placeHolderStyle)) {
var reader = scope.ResolveNamed <ISchemaReader>(Process.Connections.First().Key);
return(Process.Entities.Count == 1 ? reader.Read(Process.Entities.First()) : reader.Read());
}
}
public Schema Read(Entity entity)
{
if (Process == null)
{
_host.Error("RunTimeSchemaReader executed without a Process");
return(new Schema());
}
using (var scope = DefaultContainer.Create(Process, _host.Logger, _placeHolderStyle)) {
var reader = scope.ResolveNamed <ISchemaReader>(Process.Connections.First(c => c.Provider != Constants.DefaultSetting).Key);
return(reader.Read(entity));
}
}
public void Start()
{
using (var outer = ConfigurationContainer.Create(_options.Arrangement, _logger, null, _options.PlaceHolderStyle)) {
var process = outer.Resolve <Configuration.Process>();
var log = new PipelineContext(_logger, process);
if (process.Entities.Any(e => process.Connections.First(c => c.Name == e.Connection).Provider != Constants.DefaultSetting && !e.Fields.Any(f => f.Input)))
{
log.Debug(() => "Detecting schema...");
if (_schemaHelper.Help(process))
{
process.Check();
}
}
if (_options.Mode != null && _options.Mode.ToLower() == "check")
{
SimplifyForOutput(process);
Console.WriteLine(process.Serialize());
return;
}
if (_options.Mode != "default")
{
process.Mode = _options.Mode;
}
foreach (var warning in process.Warnings())
{
log.Debug(() => warning);
}
if (process.Errors().Any())
{
foreach (var error in process.Errors())
{
log.Error(error);
}
log.Error("The configuration errors must be fixed before this job will run.");
log.Logger.Clear();
return;
}
using (var inner = DefaultContainer.Create(process, _logger, _options.PlaceHolderStyle)) {
inner.Resolve <IProcessController>().Execute();
}
}
}
public ActionResponse Execute()
{
var response = new ActionResponse();
if (!_process.Enabled)
{
response.Code = 503;
response.Message = "Process is disabled.";
return(response);
}
using (var scope = DefaultContainer.Create(_process, _context?.Logger)) {
scope.Resolve <IProcessController>().Execute();
}
return(response);
}
private static IAction SwitchAction(IComponentContext ctx, Process process, Configuration.Action action)
{
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process);
switch (action.Type)
{
case "log":
return(new LogAction(context, action));
case "wait":
case "sleep":
return(new WaitAction(action));
case "tfl":
var cfg = string.IsNullOrEmpty(action.Url) ? action.File : action.Url;
if (string.IsNullOrEmpty(cfg) && !string.IsNullOrEmpty(action.Body))
{
cfg = action.Body;
}
var root = ctx.Resolve <Process>(new NamedParameter("cfg", cfg));
foreach (var warning in root.Warnings())
{
context.Warn(warning);
}
if (root.Errors().Any())
{
context.Error($"TFL Pipeline Action '{cfg.Left(30)}'... has errors!");
foreach (var error in root.Errors())
{
context.Error(error);
}
return(new NullAction());
}
return(new PipelineAction(DefaultContainer.Create(root, ctx.Resolve <IPipelineLogger>(), action.PlaceHolderStyle)));
case "exit":
return(new ExitAction(context, action));
default:
context.Error("{0} action is not registered.", action.Type);
return(new NullAction());
}
}
public void Test1()
{
const string xml = @"
<add name='p1' mode='meta'>
<connections>
<add name='input' provider='sqlserver' server='localhost' database='TflNorthWind' />
</connections>
<entities>
<add name='e1' query='select OrderDetailsDiscount, OrderDetailsOrderID from NorthWindStar;' />
</entities>
</add>";
var builder = new ContainerBuilder();
builder.Register <ISchemaReader>((ctx, p) => {
var process = p.TypedAs <Process>();
var context = ctx.IsRegistered <IContext>() ? ctx.Resolve <IContext>() : new PipelineContext(new NullLogger(), process);
return(new RunTimeSchemaReader(process, context));
}).As <ISchemaReader>();
builder.RegisterModule(new ShorthandTransformModule());
builder.RegisterModule(new RootModule());
using (var scope = builder.Build().BeginLifetimeScope()) {
var process = scope.Resolve <Process>(new NamedParameter("cfg", xml));
if (process.Errors().Any())
{
foreach (var error in process.Errors())
{
System.Diagnostics.Trace.WriteLine(error);
}
throw new Exception(string.Join(System.Environment.NewLine, process.Errors()));
}
using (var s = DefaultContainer.Create(process, new DebugLogger(LogLevel.Debug), "@()")) {
var rows = s.Resolve <IProcessController>().Read().ToArray();
Assert.AreEqual(2155, rows.Length);
}
}
}
public void Condense()
{
const string xml = @"
<add name='TestProcess'>
<entities>
<add name='TestData' pipeline='linq'>
<rows>
<add Field1='Some duplicates spa ces' />
<add Field1='O n e S p a c ee' />
</rows>
<fields>
<add name='Field1' />
</fields>
<calculated-fields>
<add name='CondenseField1' t='copy(Field1).condense()' />
<add name='DuplicateE' t='copy(Field1).condense(e)' />
</calculated-fields>
</add>
</entities>
</add>";
var logger = new TraceLogger(LogLevel.Debug);
using (var outer = ConfigurationContainer.Create(xml, logger)) {
var process = outer.Resolve <Process>();
using (var inner = DefaultContainer.Create(process, logger, "@()")) {
var results = inner.Resolve <IProcessController>().Read().ToArray();
var condensed = process.Entities.First().CalculatedFields.First();
var duplicate = process.Entities.First().CalculatedFields.Last();
Assert.AreEqual("Some duplicates spa ces", results[0][condensed]);
Assert.AreEqual("O n e S p a c ee", results[1][condensed]);
Assert.AreEqual("O n e S p a c e", results[1][duplicate]);
}
}
}
/// <summary>
/// 创建实例
/// </summary>
/// <typeparam name="T">对象类型</typeparam>
/// <param name="name">服务名称</param>
/// <returns></returns>
public static T Create <T>(string name = null)
{
return(DefaultContainer.Create <T>(name));
}
public void Execute()
{
#region cfg
var xml = @"
<add name='CombineInput' mode='default'>
<entities>
<add name='Feeding'>
<fields>
<add name='Id' type='byte' primary-key='true' />
<add name='When' type='datetime' />
<add name='PetId' type='byte' />
</fields>
<rows>
<add Id='1' When='2017-02-03 9:00 AM' PetId='1' />
<add Id='2' When='2017-02-03 9:05 AM' PetId='2' />
<add Id='3' When='2017-02-03 1:30 PM' PetId='1' />
<add Id='4' When='2017-02-03 5:00 PM' PetId='1' />
<add Id='5' When='2017-02-03 5:03 PM' PetId='2' />
</rows>
</add>
<add name='Pet'>
<fields>
<add name='Id' type='byte' primary-key='true' />
<add name='Name' />
</fields>
<rows>
<add Id='1' Name='Lucy' />
<add Id='2' Name='Hammond' />
</rows>
</add>
</entities>
<relationships>
<add left-entity='Feeding' left-field='PetId' right-entity='Pet' right-field='Id' />
</relationships>
</add>";
#endregion
var provider = "sqlite";
var ext = "sqlite3";
var process = ProcessFactory.Create(xml, @"Shorthand.xml");
if (!process.Errors().Any())
{
var originalOutput = process.Output().Clone();
if (process.Entities.Count > 1 && !process.OutputIsRelational())
{
process.Output().Provider = provider;
var file = new FileInfo(Path.Combine(Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "Transformalize"), process.Name + "." + ext));
process.Output().File = file.FullName;
Console.WriteLine(file.FullName);
process.Flatten = provider == "sqlce";
process.Mode = "init";
if (!file.Exists)
{
if (!Directory.Exists(file.DirectoryName))
{
Directory.CreateDirectory(Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "Transformalize"));
}
}
using (var scope = DefaultContainer.Create(process, new DebugLogger())) {
scope.Resolve <IProcessController>().Execute();
}
var threshold = process.Entities.Min(e => e.BatchId) - 1;
var reversed = new Process {
Name = process.Name,
ReadOnly = true,
Connections = new List <Connection>(2)
{
new Connection {
Name = "input", Provider = provider, File = file.FullName
},
originalOutput
},
Entities = new List <Entity>(1)
{
new Entity {
Name = provider == "sqlce" ? process.Flat : process.Star,
CalculateHashCode = false,
Connection = "input",
Fields = process.GetStarFields().SelectMany(f => f).Select(field => new Field {
Name = field.Alias,
Alias = field.Alias,
Type = field.Type,
Input = true,
PrimaryKey = field.Name == Transformalize.Constants.TflKey
}).ToList(),
Filter = new List <Filter> {
new Filter {
Field = Transformalize.Constants.TflBatchId,
Operator = "greaterthan",
Value = threshold.ToString()
}
}
}
}
};
reversed.Check();
if (reversed.Errors().Any())
{
foreach (var error in reversed.Errors())
{
Console.WriteLine(error);
}
}
using (var scope = DefaultContainer.Create(reversed, new DebugLogger())) {
scope.Resolve <IProcessController>().Execute();
if (originalOutput.Provider == "internal")
{
process.Rows = reversed.Entities.First().Rows;
}
Assert.AreEqual(5, process.Rows.Count);
}
}
}
}
protected override void Load(ContainerBuilder builder)
{
if (_process == null)
{
return;
}
// connections
foreach (var connection in _process.Connections.Where(c => c.Provider == "excel"))
{
builder.Register <ISchemaReader>(ctx => {
/* file and excel are different, have to load the content and check it to determine schema */
var fileInfo = new FileInfo(Path.IsPathRooted(connection.File) ? connection.File : Path.Combine(AppDomain.CurrentDomain.BaseDirectory, connection.File));
var context = ctx.ResolveNamed <IConnectionContext>(connection.Key);
var cfg = new ExcelInspection(context, fileInfo, 100).Create();
var process = ctx.Resolve <Process>();
process.Load(cfg);
foreach (var warning in process.Warnings())
{
context.Warn(warning);
}
if (process.Errors().Any())
{
foreach (var error in process.Errors())
{
context.Error(error);
}
return(new NullSchemaReader());
}
var container = DefaultContainer.Create(process, ctx.Resolve <IPipelineLogger>(), ctx.ResolveNamed <string>("placeHolderStyle"));
return(new SchemaReader(context, new RunTimeRunner(context, container), process));
}).Named <ISchemaReader>(connection.Key);
}
// Entity input
foreach (var entity in _process.Entities.Where(e => _process.Connections.First(c => c.Name == e.Connection).Provider == "excel"))
{
// input version detector
builder.RegisterType <NullInputProvider>().Named <IInputProvider>(entity.Key);
// input reader
builder.Register <IRead>(ctx => {
var input = ctx.ResolveNamed <InputContext>(entity.Key);
var rowFactory = ctx.ResolveNamed <IRowFactory>(entity.Key, new NamedParameter("capacity", input.RowCapacity));
switch (input.Connection.Provider)
{
case "excel":
return(new ExcelReader(input, rowFactory));
default:
return(new NullReader(input, false));
}
}).Named <IRead>(entity.Key);
}
if (_process.Output().Provider == "excel")
{
// PROCESS OUTPUT CONTROLLER
builder.Register <IOutputController>(ctx => new NullOutputController()).As <IOutputController>();
foreach (var entity in _process.Entities)
{
builder.Register <IOutputController>(ctx => new NullOutputController()).Named <IOutputController>(entity.Key);
// ENTITY WRITER
builder.Register <IWrite>(ctx => {
var output = ctx.ResolveNamed <OutputContext>(entity.Key);
switch (output.Connection.Provider)
{
case "excel":
return(new ExcelWriter(output));
default:
return(new NullWriter(output));
}
}).Named <IWrite>(entity.Key);
}
}
}
private static IAction SwitchAction(IComponentContext ctx, Process process, Action action)
{
Connection connection;
var context = new PipelineContext(ctx.Resolve <IPipelineLogger>(), process);
switch (action.Type)
{
case "copy":
return(action.InTemplate ? (IAction)
new ContentToFileAction(context, action) :
new FileCopyAction(context, action));
case "move":
return(new FileMoveAction(context, action));
case "archive":
return(new FileArchiveAction(context, action));
case "replace":
return(new FileReplaceAction(context, action));
case "print":
return(new PrintAction(action));
case "log":
return(new LogAction(context, action));
case "web":
return(new WebAction(context, action));
case "form-commands":
connection = process.Connections.First(c => c.Name == action.Connection);
return(new AdoEntityFormCommands(context, action, ctx.ResolveNamed <IConnectionFactory>(connection.Key)));
case "wait":
case "sleep":
return(new WaitAction(action));
case "tfl":
var cfg = string.IsNullOrEmpty(action.Url) ? action.File : action.Url;
if (string.IsNullOrEmpty(cfg) && !string.IsNullOrEmpty(action.Body))
{
cfg = action.Body;
}
var root = ctx.Resolve <Process>(new NamedParameter("cfg", cfg));
foreach (var warning in root.Warnings())
{
context.Warn(warning);
}
if (root.Errors().Any())
{
context.Error($"TFL Pipeline Action '{cfg.Left(15) + "..." + cfg.Right(15)}' has errors!");
foreach (var error in root.Errors())
{
context.Error(error);
}
return(new NullAction());
}
return(new PipelineAction(DefaultContainer.Create(root, ctx.Resolve <IPipelineLogger>(), action.PlaceHolderStyle)));
case "run":
connection = process.Connections.First(c => c.Name == action.Connection);
switch (connection.Provider)
{
case "mysql":
case "postgresql":
case "sqlite":
case "sqlserver":
return(new AdoRunAction(context, action, ctx.ResolveNamed <IConnectionFactory>(connection.Key), new DefaultReader(new FileReader(), new WebReader())));
default:
context.Error("{0} provider is not registered for run action.", connection.Provider);
return(new NullAction());
}
case "open":
return(new OpenAction(action));
case "exit":
return(new ExitAction(context, action));
default:
context.Error("{0} action is not registered.", action.Type);
return(new NullAction());
}
}
public static ActionResponse Execute(Process process, string placeHolderStyle = "@()")
{
var container = DefaultContainer.Create(process, new DebugLogger(), placeHolderStyle);
return(new PipelineAction(container).Execute());
}