private static void CheckMemory(Process process, IPipelineLogger logger)
{
if (string.IsNullOrEmpty(process.MaxMemory))
{
return;
}
var context = new PipelineContext(logger, process);
GC.Collect();
GC.WaitForPendingFinalizers();
var currentBytes = System.Diagnostics.Process.GetCurrentProcess().WorkingSet64.Bytes();
var maxMemory = ByteSize.Parse(process.MaxMemory);
if (maxMemory.CompareTo(currentBytes) < 0)
{
context.Error(
$"Process exceeded {maxMemory.Megabytes.ToString("#.0")} Mb. Current memory is {currentBytes.Megabytes.ToString("#.0")} Mb!");
Environment.Exit(1);
}
else
{
context.Info(
$"The process is using {currentBytes.Megabytes.ToString("#.0")} Mb of it's max {maxMemory.Megabytes.ToString("#.0")} Mb allowed.");
}
}
public PipelineContext(
IPipelineLogger logger,
Process process = null,
Entity entity = null,
Field field = null,
Transform transform = null
)
{
ForLog = new object[4];
Logger = logger;
Process = process ?? new Process {
Name = "Process"
}.WithDefaults();
Entity = entity ?? new Entity {
Name = string.Empty, Alias = string.Empty
}.WithDefaults();
Field = field ?? new Field {
Name = string.Empty, Alias = string.Empty
}.WithDefaults();
Transform = transform ?? new Transform {
Method = string.Empty
}.WithDefaults();
Key = Process.Name + Entity.Key + Field.Alias + Transform.Method + Transform.Index;
ForLog[0] = Process.Name.PadRight(Process.LogLimit, ' ').Left(Process.LogLimit);
ForLog[1] = Entity.Alias.PadRight(Process.EntityLogLimit, ' ').Left(Process.EntityLogLimit);
ForLog[2] = Field.Alias.PadRight(Process.FieldLogLimit, ' ').Left(Process.FieldLogLimit);
ForLog[3] = Transform.Method.PadRight(Process.TransformLogLimit, ' ').Left(Process.TransformLogLimit);
}
public static ILifetimeScope Create(Process process, IPipelineLogger logger)
{
if (process.OutputIsConsole())
{
logger.SuppressConsole();
}
var builder = new ContainerBuilder();
builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance();
builder.RegisterCallback(new RootModule().Configure);
builder.RegisterCallback(new ContextModule(process).Configure);
// providers
builder.RegisterCallback(new AdoModule(process).Configure);
builder.RegisterCallback(new InternalModule(process).Configure);
builder.RegisterCallback(new FileModule(process).Configure);
builder.RegisterCallback(new ExcelModule(process).Configure);
builder.RegisterCallback(new EntityPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessControlModule(process).Configure);
return(builder.Build().BeginLifetimeScope());
}
public TransformModule(Process process, HashSet <string> methods, ShorthandRoot shortHand, IPipelineLogger logger)
{
_process = process;
_logger = logger;
_methods = methods;
_shortHand = shortHand;
}
public TransformModule(Process process, IPipelineLogger logger)
{
_process = process;
_logger = logger;
_methods = new HashSet <string>();
_shortHand = new ShorthandRoot();
}
public BaseExecutor(IPipelineLogger logger, string cfg, string mode, bool checkMemory)
{
_logger = logger;
_checkMemory = checkMemory;
Cfg = cfg;
Mode = mode;
}
public PipelineContext(
IPipelineLogger logger,
Process process = null,
Entity entity = null,
Field field = null,
Operation operation = null
)
{
ForLog = new object[4];
Logger = logger;
Process = process ?? new Process {
Name = "Process"
};
Entity = entity ?? new Entity {
Name = string.Empty, Alias = string.Empty
};
Field = field ?? new Field {
Name = string.Empty, Alias = string.Empty
};
Operation = operation ?? new Operation {
Method = string.Empty
};
Key = Process.Name + Entity.Key + Field.Alias + Operation.Method + Operation.Index;
ForLog[0] = Process.Name.PadRight(Process.LogLimit, ' ').Left(Process.LogLimit);
ForLog[1] = Entity.Alias.PadRight(Process.EntityLogLimit, ' ').Left(Process.EntityLogLimit);
ForLog[2] = Field.Alias.PadRight(Process.FieldLogLimit, ' ').Left(Process.FieldLogLimit);
ForLog[3] = Operation.Method.PadRight(Process.TransformLogLimit, ' ').Left(Process.TransformLogLimit);
}
public ValidateBuilder(Process process, ContainerBuilder builder, IPipelineLogger logger)
{
_process = process;
_logger = logger;
_methods = new HashSet <string>();
_shortHand = new ShorthandRoot();
_builder = builder;
_validators = new List <ValidatorHolder>();
}
public TransformBuilder(Process process, ContainerBuilder builder, IPipelineLogger logger)
{
_process = process;
_builder = builder;
_logger = logger;
_methods = new HashSet <string>();
_shortHand = new ShorthandRoot();
_transforms = new List <TransformHolder>();
}
public Bootstrapper(IPipelineLogger logger = null)
{
var builder = new ContainerBuilder();
builder.RegisterModule(new JunkModule(logger));
builder.RegisterModule(new ImportModule());
builder.RegisterModule(new PageModule());
_scope = builder.Build().BeginLifetimeScope();
}
public TransformBuilder(Process process, ContainerBuilder builder, HashSet <string> methods, ShorthandRoot shortHand, List <TransformHolder> transforms, IPipelineLogger logger)
{
_process = process;
_builder = builder;
_logger = logger;
_methods = methods;
_shortHand = shortHand;
_transforms = transforms;
}
public ValidateBuilder(Process process, ContainerBuilder builder, HashSet <string> methods, ShorthandRoot shortHand, List <ValidatorHolder> validators, IPipelineLogger logger)
{
_process = process;
_logger = logger;
_methods = methods;
_shortHand = shortHand;
_builder = builder;
_validators = validators;
}
public RunTimeDataReader(
IPipelineLogger logger,
IAppDataFolder appDataFolder,
ITemplateProcessor templateProcessor,
INotifier notifier)
{
_logger = logger;
_appDataFolder = appDataFolder;
_templateProcessor = templateProcessor;
_notifier = notifier;
}
public static ILifetimeScope Create(Process process, IPipelineLogger logger, string placeHolderStyle)
{
if (process.OutputIsConsole())
{
logger.SuppressConsole();
}
var builder = new ContainerBuilder();
builder.Register(ctx => placeHolderStyle).Named <string>("placeHolderStyle");
builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance();
builder.RegisterCallback(new TransformModule().Configure);
builder.RegisterCallback(new ShorthandTransformModule().Configure);
builder.RegisterCallback(new ValidateModule().Configure);
builder.RegisterCallback(new ShorthandValidateModule().Configure);
builder.RegisterCallback(new RootModule().Configure);
builder.RegisterCallback(new ContextModule(process).Configure);
// providers
builder.RegisterCallback(new AdoModule(process).Configure);
builder.RegisterCallback(new LuceneModule(process).Configure);
builder.RegisterCallback(new SolrModule(process).Configure);
builder.RegisterCallback(new ElasticModule(process).Configure);
builder.RegisterCallback(new InternalModule(process).Configure);
builder.RegisterCallback(new ConsoleModule(process).Configure);
builder.RegisterCallback(new FileModule(process).Configure);
builder.RegisterCallback(new GeoJsonModule(process).Configure);
builder.RegisterCallback(new KmlModule(process).Configure);
builder.RegisterCallback(new FolderModule(process).Configure);
builder.RegisterCallback(new DirectoryModule(process).Configure);
builder.RegisterCallback(new ExcelModule(process).Configure);
builder.RegisterCallback(new WebModule(process).Configure);
builder.RegisterCallback(new SSASModule(process).Configure);
builder.RegisterCallback(new RethinkDBModule(process).Configure);
// template providers
builder.RegisterCallback(new WordModule(process).Configure);
builder.RegisterCallback(new RazorModule(process).Configure);
// etc
builder.RegisterCallback(new MapModule(process).Configure);
builder.RegisterCallback(new TemplateModule(process).Configure);
builder.RegisterCallback(new ActionModule(process).Configure);
builder.RegisterCallback(new EntityPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessControlModule(process).Configure);
return(builder.Build().BeginLifetimeScope());
}
public static ILifetimeScope Create(string cfg, IPipelineLogger logger, Dictionary <string, string> parameters = null, string placeHolderStyle = "@()")
{
var builder = new ContainerBuilder();
builder.Register(c => logger).As <IPipelineLogger>();
builder.Register(c => placeHolderStyle).Named <string>("placeHolderStyle");
builder.RegisterModule(new TransformModule(new Process {
Name = "ConfigurationContainer"
}, logger));
builder.RegisterModule(new ShorthandTransformModule());
builder.RegisterModule(new ValidateModule());
builder.RegisterModule(new ShorthandValidateModule());
builder.RegisterModule(new RootModule());
return(builder.Build());
}
public static ILifetimeScope Create(string cfg, IPipelineLogger logger, Dictionary <string, string> parameters = null, string placeHolderStyle = "@()")
{
var builder = new ContainerBuilder();
builder.Register(c => cfg).Named <string>(Cfg);
builder.Register(c => logger).As <IPipelineLogger>();
builder.Register(c => placeHolderStyle).Named <string>("placeHolderStyle");
// these are called here to register short-hand (t and v attributes)
builder.RegisterModule(new TransformModule(new Process {
Name = Cfg
}, logger));
builder.RegisterModule(new ValidateModule(new Process {
Name = Cfg
}, logger));
builder.RegisterModule(new RootModule());
return(builder.Build());
}
public PipelineContext(
IPipelineLogger logger,
Process process,
Entity entity = null,
Field field = null,
Transform transform = null
) {
ForLog = new object[5];
Logger = logger;
Activity = PipelineActivity.Transform;
Process = process;
Entity = entity ?? process.GetValidatedOf<Entity>(e => { e.Name = string.Empty; });
Field = field ?? process.GetValidatedOf<Field>(f => { f.Name = string.Empty; });
Transform = transform ?? process.GetDefaultOf<Transform>(t => { t.Method = string.Empty; });
ForLog[0] = process.Name.PadRight(process.LogLimit, ' ').Left(process.LogLimit);
ForLog[1] = Entity.Alias.PadRight(process.EntityLogLimit, ' ').Left(process.EntityLogLimit);
ForLog[2] = ' ';
ForLog[3] = Field.Alias.PadRight(process.FieldLogLimit, ' ').Left(process.FieldLogLimit);
ForLog[4] = Transform.Method.PadRight(process.TransformLogLimit, ' ').Left(process.TransformLogLimit);
}
public PipelineContext(
IPipelineLogger logger,
Process process,
Entity entity = null,
Field field = null,
Transform transform = null
)
{
ForLog = new object[5];
Logger = logger;
Activity = PipelineActivity.Transform;
Process = process;
Entity = entity ?? process.GetValidatedOf <Entity>(e => { e.Name = string.Empty; });
Field = field ?? process.GetValidatedOf <Field>(f => { f.Name = string.Empty; });
Transform = transform ?? process.GetDefaultOf <Transform>(t => { t.Method = string.Empty; });
ForLog[0] = process.Name.PadRight(process.LogLimit, ' ').Left(process.LogLimit);
ForLog[1] = Entity.Alias.PadRight(process.EntityLogLimit, ' ').Left(process.EntityLogLimit);
ForLog[2] = ' ';
ForLog[3] = Field.Alias.PadRight(process.FieldLogLimit, ' ').Left(process.FieldLogLimit);
ForLog[4] = Transform.Method.PadRight(process.TransformLogLimit, ' ').Left(process.TransformLogLimit);
}
public static ILifetimeScope Create(Process process, IPipelineLogger logger)
{
if (process.OutputIsConsole())
{
logger.SuppressConsole();
}
var builder = new ContainerBuilder();
builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance();
builder.RegisterCallback(new RootModule(process.Shorthand).Configure);
builder.RegisterCallback(new ContextModule(process).Configure);
// providers
builder.RegisterCallback(new AdoModule(process).Configure);
builder.RegisterCallback(new LuceneModule(process).Configure);
builder.RegisterCallback(new SolrModule(process).Configure);
builder.RegisterCallback(new ElasticModule(process).Configure);
builder.RegisterCallback(new InternalModule(process).Configure);
builder.RegisterCallback(new FileModule(process).Configure);
builder.RegisterCallback(new GeoJsonModule(process).Configure);
builder.RegisterCallback(new KmlModule(process).Configure);
builder.RegisterCallback(new FolderModule(process).Configure);
builder.RegisterCallback(new DirectoryModule(process).Configure);
builder.RegisterCallback(new ExcelModule(process).Configure);
builder.RegisterCallback(new WebModule(process).Configure);
builder.RegisterCallback(new MapModule(process).Configure);
builder.RegisterCallback(new TemplateModule(process).Configure);
builder.RegisterCallback(new ActionModule(process).Configure);
builder.RegisterCallback(new TransformModule(process).Configure);
builder.RegisterCallback(new EntityPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessControlModule(process).Configure);
return(builder.Build().BeginLifetimeScope());
}
private static void CheckMemory(Process process, IPipelineLogger logger)
{
if (string.IsNullOrEmpty(process.MaxMemory))
{
return;
}
var context = new PipelineContext(logger, process);
GC.Collect();
GC.WaitForPendingFinalizers();
var bytes = System.Diagnostics.Process.GetCurrentProcess().WorkingSet64.Bytes();
if (bytes.Megabytes > MaxBytes.Megabytes)
{
if (MaxBytes.Megabytes > 0.0)
{
context.Warn($"Process memory has increased from {MaxBytes.Megabytes:#.0} to {bytes.Megabytes:#.0}.");
}
MaxBytes = bytes;
}
var maxMemory = ByteSize.Parse(process.MaxMemory);
if (maxMemory.CompareTo(bytes) < 0)
{
context.Error($"Process exceeded {maxMemory.Megabytes:#.0} Mb. Current memory is {bytes.Megabytes:#.0} Mb!");
context.Error($"Process is exiting after running {RunCount} time{RunCount.Plural()}.");
Environment.Exit(1);
}
else
{
context.Info($"Process is using {bytes.Megabytes:#.0} Mb of it's max {maxMemory.Megabytes:#.0} Mb allowed.");
context.Info($"Process has run {RunCount} time{RunCount.Plural()}.");
}
}
/// <summary>
/// Hides the work of creating the context and input for each method
/// </summary>
/// <param name="context"></param>
/// <param name="entity"></param>
/// <param name="field"></param>
/// <param name="sb"></param>
private static void WriteMethods(Process process, Entity entity, Field field, StringBuilder sb, IPipelineLogger logger)
{
foreach (var transform in field.Transforms.Where(t => t.Method == "cs" || t.Method == "csharp"))
{
var tc = new PipelineContext(logger, process, entity, field, transform);
var input = process.ParametersToFields(transform.Parameters, field);
WriteMethod(tc, input, sb);
}
}
public ShorthandModule(
IPipelineLogger logger
)
{
_logger = logger;
}
public TransformModule(Process process, IPipelineLogger logger)
{
_process = process;
_logger = logger;
}
public JunkModule(Request jr, IPipelineLogger logger = null)
{
_jr = jr;
_logger = logger;
}
public NowScheduler(Options options, ISchemaHelper schemaHelper, IPipelineLogger logger)
{
_options = options;
_schemaHelper = schemaHelper;
_logger = logger;
}
public static ILifetimeScope Create(Process process, IPipelineLogger logger, string placeHolderStyle)
{
var loadContext = new PipelineContext(logger, process);
if (process.OutputIsConsole())
{
logger.SuppressConsole();
}
var builder = new ContainerBuilder();
builder.Properties["Process"] = process;
builder.Register(ctx => placeHolderStyle).Named <string>("placeHolderStyle");
builder.RegisterInstance(logger).As <IPipelineLogger>().SingleInstance();
/* this stuff is loaded (again) because tfl actions can create processes, which will need short-hand to expand configuration in advance */
builder.RegisterCallback(new TransformModule(process, logger).Configure);
builder.RegisterCallback(new ValidateModule(process, logger).Configure);
builder.RegisterCallback(new RootModule().Configure);
builder.RegisterCallback(new ContextModule(process).Configure);
// provider loading section
var providers = new HashSet <string>(process.Connections.Select(c => c.Provider).Distinct(), StringComparer.OrdinalIgnoreCase);
builder.RegisterCallback(new InternalModule(process).Configure);
builder.RegisterCallback(new FileModule(process).Configure);
if (providers.Contains("console"))
{
builder.RegisterCallback(new ConsoleModule(process).Configure);
}
if (providers.Contains("kml"))
{
builder.RegisterCallback(new KmlModule(process).Configure);
}
if (providers.Contains("filesystem"))
{
builder.RegisterCallback(new FileSystemModule(process).Configure);
}
var pluginsFolder = Path.Combine(AssemblyDirectory, "plugins");
if (Directory.Exists(pluginsFolder))
{
var assemblies = new List <Assembly>();
var files = Directory.GetFiles(pluginsFolder, "Transformalize.Provider.*.Autofac.dll", SearchOption.TopDirectoryOnly);
foreach (var file in files)
{
var info = new FileInfo(file);
var name = info.Name.ToLower().Split('.').FirstOrDefault(f => f != "dll" && f != "transformalize" && f != "provider" && f != "autofac");
// temporary hack
if (name.StartsWith("amazonkinesis"))
{
name = name.Replace("amazonkinesis", string.Empty);
}
switch (name)
{
case "filehelpers" when(providers.Contains("file") || providers.Contains("folder")):
loadContext.Debug(() => "Loading filehelpers provider");
assemblies.Add(Assembly.LoadFile(new FileInfo(file).FullName));
break;
case "ado":
loadContext.Debug(() => "Loading ADO provider");
assemblies.Add(Assembly.LoadFile(new FileInfo(file).FullName));
break;
default:
if (providers.Contains(name))
{
loadContext.Debug(() => $"Loading {name} provider.");
var assembly = Assembly.LoadFile(new FileInfo(file).FullName);
assemblies.Add(assembly);
}
else
{
loadContext.Debug(() => $"Loading {name} isn't necessary for this arrangement.");
}
break;
}
}
if (assemblies.Any())
{
builder.RegisterAssemblyModules(assemblies.ToArray());
}
}
// etc
builder.RegisterCallback(new EntityPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessPipelineModule(process).Configure);
builder.RegisterCallback(new ProcessControlModule(process).Configure);
return(builder.Build().BeginLifetimeScope());
}
public ILifetimeScope CreateScope(string cfg, IPipelineLogger logger, IDictionary <string, string> parameters = null, string placeHolderStyle = "@[]")
{
if (placeHolderStyle == null || placeHolderStyle.Length != 3)
{
throw new ArgumentException("The placeHolderStyle parameter must be three characters (e.g. @[], or @())");
}
if (parameters == null)
{
parameters = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
}
var builder = new ContainerBuilder();
builder.Register((ctx) => logger).As <IPipelineLogger>();
builder.Register <IReader>(c => new DefaultReader(new FileReader(), new WebReader())).As <IReader>();
// register short-hand for t attribute
var transformModule = new TransformModule(new Process {
Name = "TransformShorthand"
}, _methods, _shortHand, logger);
foreach (var t in _transforms)
{
transformModule.AddTransform(t);
}
builder.RegisterModule(transformModule);
// register short-hand for v attribute
var validateModule = new ValidateModule(new Process {
Name = "ValidateShorthand"
}, _methods, _shortHand, logger);
foreach (var v in _validators)
{
validateModule.AddValidator(v);
}
builder.RegisterModule(validateModule);
// register the validator short hand
builder.Register((c, p) => _shortHand).Named <ShorthandRoot>(ValidateModule.FieldsName).InstancePerLifetimeScope();
builder.Register((c, p) => new ShorthandCustomizer(c.ResolveNamed <ShorthandRoot>(ValidateModule.FieldsName), new[] { "fields", "calculated-fields", "calculatedfields" }, "v", "validators", "method")).Named <IDependency>(ValidateModule.FieldsName).InstancePerLifetimeScope();
// register the transform short hand
builder.Register((c, p) => _shortHand).Named <ShorthandRoot>(TransformModule.FieldsName).InstancePerLifetimeScope();
builder.Register((c, p) => _shortHand).Named <ShorthandRoot>(TransformModule.ParametersName).InstancePerLifetimeScope();
builder.Register((c, p) => new ShorthandCustomizer(c.ResolveNamed <ShorthandRoot>(TransformModule.FieldsName), new[] { "fields", "calculated-fields", "calculatedfields" }, "t", "transforms", "method")).Named <IDependency>(TransformModule.FieldsName).InstancePerLifetimeScope();
builder.Register((c, p) => new ShorthandCustomizer(c.ResolveNamed <ShorthandRoot>(TransformModule.ParametersName), new[] { "parameters" }, "t", "transforms", "method")).Named <IDependency>(TransformModule.ParametersName).InstancePerLifetimeScope();
#if PLUGINS
// the shorthand registrations are stored in the builder's properties for plugins to add to
builder.Properties["ShortHand"] = _shortHand;
builder.Properties["Methods"] = _methods;
#endif
foreach (var module in _modules)
{
builder.RegisterModule(module);
}
builder.Register((c, p) => _methods).Named <HashSet <string> >("Methods").InstancePerLifetimeScope();
builder.Register((c, p) => _shortHand).As <ShorthandRoot>().InstancePerLifetimeScope();
builder.Register(ctx => {
var transformed = TransformParameters(ctx, cfg);
var dependancies = new List <IDependency>();
dependancies.Add(ctx.Resolve <IReader>());
dependancies.AddRange(_dependencies);
dependancies.Add(new ParameterModifier(new PlaceHolderReplacer(placeHolderStyle[0], placeHolderStyle[1], placeHolderStyle[2]), "parameters", "name", "value"));
dependancies.Add(ctx.ResolveNamed <IDependency>(TransformModule.FieldsName));
dependancies.Add(ctx.ResolveNamed <IDependency>(TransformModule.ParametersName));
dependancies.Add(ctx.ResolveNamed <IDependency>(ValidateModule.FieldsName));
var process = new Process(transformed ?? cfg, parameters, dependancies.ToArray());
if (process.Errors().Any())
{
var c = new PipelineContext(logger, new Process()
{
Name = "Errors"
});
c.Error("The configuration has errors.");
foreach (var error in process.Errors())
{
c.Error(error);
}
}
return(process);
}).As <Process>().InstancePerDependency();
return(builder.Build().BeginLifetimeScope());
}
public JunkModule(IPipelineLogger logger = null)
{
_logger = logger;
// sometimes you just don't have the junk request yet...
}
public ValidateModule(Process process, IPipelineLogger logger)
{
_process = process;
_logger = logger;
}
public ContextFactory(Process process, IPipelineLogger logger)
{
_process = process;
_logger = logger;
}
public AutofacJunkBootstrapperFactory(IPipelineLogger logger = null)
{
_logger = logger;
}
