public static IUndupAction CreateAction(ImportEngine engine, IPostProcessor processor, XmlNode node)
{
String type = node.ReadStr("@type");
if ("add".Equals(type, StringComparison.OrdinalIgnoreCase))
{
return(new UndupNumericAddAction(processor, node));
}
if ("max".Equals(type, StringComparison.OrdinalIgnoreCase))
{
return(new UndupNumericMaxAction(processor, node));
}
if ("min".Equals(type, StringComparison.OrdinalIgnoreCase))
{
return(new UndupNumericMinAction(processor, node));
}
if ("mean".Equals(type, StringComparison.OrdinalIgnoreCase))
{
return(new UndupNumericMeanAction(processor, node));
}
if ("count".Equals(type, StringComparison.OrdinalIgnoreCase))
{
return(new UndupCountAction(processor, node));
}
if ("script".Equals(type, StringComparison.OrdinalIgnoreCase))
{
return(new UndupScriptAction(engine, processor, node));
}
throw new BMException("Unrecognized type [{0}] for a post process action.", type);
//TODO make this more generic
}
public MapReduceProcessor(ImportEngine engine, XmlNode node) : base(engine, node)
{
maxNullIndex = node.ReadInt("@max_null_index", -1);
fanOut = node.ReadInt("@fan_out", 100);
if (fanOut <= 0)
{
throw new BMNodeException(node, "Count should be > 0.");
}
if (node.ReadInt("write/@maxparallel", 1) > 0)
{
bufferSize = node.ReadInt("write/@buffer", 100);
}
readMaxParallel = node.ReadInt("read/@maxparallel", 1);
directory = node.ReadStr("dir/@name", null);
if (directory != null)
{
directory = engine.Xml.CombinePath(directory);
keepFiles = node.ReadBool("dir/@keepfiles", false);
compress = node.ReadBool("dir/@compress", true);
}
List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("sorter"), "key", false);
sorter = JComparer.Create(list);
XmlNode hashNode = node.SelectSingleNode("hasher");
if (hashNode == null)
{
hasher = sorter;
}
else
{
hasher = sorter.Clone(hashNode.ReadStr("@from_sort", null));
if (hasher == null)
{
list = KeyAndType.CreateKeyList(hashNode, "key", true);
hasher = JComparer.Create(list);
}
}
XmlNode undupNode = node.SelectSingleNode("undupper");
if (undupNode != null)
{
undupper = sorter.Clone(undupNode.ReadStr("@from_sort", null));
if (undupper == null)
{
list = KeyAndType.CreateKeyList(undupNode, "key", true);
undupper = JComparer.Create(list);
}
XmlNode actionsNode = undupNode.SelectSingleNode("actions");
if (actionsNode != null)
{
undupActions = new UndupActions(engine, this, actionsNode);
}
}
}
public PipelineContext(PipelineContext ctx, DatasourceAdmin ds, DatasourceReport report)
{
var eng = ctx.ImportEngine;
Switches = ctx.Switches;
NewLastUpdated = eng.StartTimeUtc;
ImportEngine = eng;
RunAdministrations = eng.RunAdministrations;
DatasourceAdmin = ds;
DatasourceReport = report;
Pipeline = ds.Pipeline;
ImportLog = eng.ImportLog.Clone(ds.Name);
DebugLog = eng.DebugLog.Clone(ds.Name);
ErrorLog = eng.ErrorLog.Clone(ds.Name);
MissedLog = eng.MissedLog.Clone(ds.Name);
ImportFlags = eng.ImportFlags;
LogAdds = (ds.LogAdds > 0) ? ds.LogAdds : eng.LogAdds;
MaxAdds = (ds.MaxAdds >= 0) ? ds.MaxAdds : eng.MaxAdds;
MaxEmits = (ds.MaxEmits >= 0) ? ds.MaxEmits : eng.MaxEmits;
if (MaxEmits < 0 && (ImportFlags & _ImportFlags.MaxAddsToMaxEmits) != 0)
{
MaxEmits = MaxAdds;
}
ImportLog.Log("Current maxAdds={0}, maxEmits={1}", MaxAdds, MaxEmits);
}
public RunAdministrationSettings(ImportEngine engine, String fn, int cap, int dump)
{
Engine = engine;
FileName = fn;
Capacity = cap;
Dump = dump;
}
public Endpoint (ImportEngine engine, XmlNode node, ImportPipeline.ActiveMode defActiveMode=0)
: base(node)
{
Engine = engine;
if (defActiveMode == 0) defActiveMode = ImportPipeline.ActiveMode.Lazy | ImportPipeline.ActiveMode.Global;
if ((defActiveMode & (ImportPipeline.ActiveMode.Local | ImportPipeline.ActiveMode.Global)) != 0)
defActiveMode |= ImportPipeline.ActiveMode.Global;
Flags = node.ReadEnum<DebugFlags>("@flags", 0);
CloseMode = node.ReadEnum("@closemode", CloseMode.Normal);
ActiveMode = node.ReadEnum("@active", defActiveMode);
switch (ActiveMode)
{
case 0:
ActiveMode = defActiveMode;
break;
case ImportPipeline.ActiveMode.False:
case ImportPipeline.ActiveMode.True:
break;
default:
if ((ActiveMode & (ImportPipeline.ActiveMode.False | ImportPipeline.ActiveMode.True)) != 0)
throw new BMNodeException (node, "ActiveMode true/false cannot be combined with other flags. ActiveMode={0}.", ActiveMode);
if ((ActiveMode & (ImportPipeline.ActiveMode.Local | ImportPipeline.ActiveMode.Global)) != 0)
ActiveMode |= defActiveMode & (ImportPipeline.ActiveMode.Local | ImportPipeline.ActiveMode.Global);
break;
}
}
public UndupScriptAction(ImportEngine engine, IPostProcessor processor, XmlNode node)
{
originalNode = node;
ScriptName = node.ReadStr("@script", null);
ScriptBody = node.ReadStr(null, null);
if (ScriptBody == null)
{
if (ScriptName == null)
{
throw new BMNodeException(node, "@script should be specified if there is no script in the body.");
}
}
else
{
ScriptBody = ScriptBody.TrimWhiteSpaceToNull();
if (ScriptBody != null && ScriptName != null)
{
throw new BMNodeException(node, "Cannot have a script in the body when @script is specified.");
}
var scriptHolder = engine.ScriptExpressions;
bodyFunc = ScriptExpressionHolder.GenerateScriptName("postprocessor_undup", processor.Name, node);
scriptHolder.AddUndupExpression(bodyFunc, ScriptBody);
}
}
public CommandEndpoint(ImportEngine engine, XmlNode node)
: base(engine, node, ActiveMode.Lazy | ActiveMode.Local)
{
XmlNodeList chlds = node.ChildNodes;// node.SelectMandatoryNodes("*");
var cmds = new List <Command>(chlds.Count);
for (int i = 0; i < chlds.Count; i++)
{
XmlNode n = chlds[i];
String name = n.Name;
Command cmd;
switch (name)
{
default: continue;
case "command":
case "exec": cmd = new ExecCommand(engine, n); break;
case "delete":
case "del": cmd = new DeleteCommand(engine, n); break;
case "copy": cmd = new CopyCommand(engine, n); break;
case "move": cmd = new MoveCommand(engine, n); break;
}
cmds.Add(cmd);
}
Commands = cmds.ToArray();
if (cmds.Count == 0)
{
engine.ImportLog.Log("Endpoint [{0}] has no commands. Resulting in a no-op.", Name);
}
}
public TopProcessor(ImportEngine engine, XmlNode node) : base(engine, node)
{
List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("sorter"), "key", true);
sorter = JComparer.Create(list);
topCount = node.ReadInt("@count");
reverse = node.ReadBool("@reverse", false);
sortAfter = node.ReadEnum("@sortafter", _SortAfter.True);
}
public Endpoints(ImportEngine engine, XmlNode collNode)
: base(collNode, "endpoint", (n) => create (engine, n), false)
{
this.engine = engine;
if (GetByName("nop", false) == null)
{
Add (new Endpoint("nop")); //always have a NOP endpoint for free
}
}
//private ImportEngine engine;
//private SortProcessor sortProcessor;
//private XmlNode actionsNode;
public UndupActions(ImportEngine engine, IPostProcessor processor, XmlNode node)
{
XmlNodeList nodes = node.SelectNodes("action");
actions = new List <IUndupAction>(nodes.Count);
foreach (XmlNode child in nodes)
{
actions.Add(CreateAction(engine, processor, child));
}
}
public PipelineContext(ImportEngine eng)
{
ImportEngine = eng;
ImportLog = eng.ImportLog;
DebugLog = eng.DebugLog;
ErrorLog = eng.ErrorLog;
MissedLog = eng.MissedLog;
ImportFlags = eng.ImportFlags;
NewLastUpdated = eng.StartTimeUtc;
RunAdministrations = eng.RunAdministrations;
Switches = new Switches(eng.SwitchesFromXml + " " + eng.Switches); //Switches are overriding
}
public ExecCommand(ImportEngine eng, XmlNode node)
: base(eng, node)
{
cmd = node.ReadStr("@cmd");
if (!cmd.StartsWith("cmd "))
{
cmd = "cmd /c " + cmd;
}
settings = new ProcessHostSettings();
settings.LogName = node.ReadStr("@log", settings.LogName);
settings.LogName = node.ReadStr("@errorlog", settings.LogName);
settings.ExeName = "cmd.exe";
}
public CsvEndpoint(ImportEngine engine, XmlNode node)
: base(engine, node, ActiveMode.Lazy | ActiveMode.Local)
{
MaxGenerations = node.ReadInt("@generations", int.MinValue);
if (MaxGenerations != int.MinValue)
{
generations = new FileGenerations();
}
String tmp = MaxGenerations == int.MinValue ? node.ReadStr("@file") : node.ReadStr("@file", null);
FileNameBase = engine.Xml.CombinePath(tmp == null ? "csvOutput" : tmp);
fileName = FileNameBase;
trim = node.ReadBool("@trim", true);
lenient = node.ReadBool("@lenient", false);
delimChar = CsvDatasource.readChar(node, "@dlm", ',');
quoteChar = CsvDatasource.readChar(node, "@quote", '"');
commentChar = CsvDatasource.readChar(node, "@comment", '#');
//Predefine field orders if requested (implies linient mode)
String fields = node.ReadStr("@fields", null);
String fieldsOrder = node.ReadStr("@fieldorder", null);
if (fields != null && fieldsOrder != null)
{
throw new BMNodeException(node, "Attributes [fields] and [fieldorder] cannot be specified together.");
}
String[] order;
if (fields != null)
{
order = fields.SplitStandard();
}
else if (fieldsOrder != null)
{
order = fieldsOrder.SplitStandard();
}
else
{
order = null;
}
if (order != null)
{
lenient = true;
foreach (var fld in order)
{
keyToIndex(fld);
}
selectiveExport = fields != null;
}
}
public TextEndpoint(ImportEngine engine, XmlNode node)
: base(engine, node, ActiveMode.Lazy | ActiveMode.Local)
{
MaxGenerations = node.ReadInt("@generations", int.MinValue);
if (MaxGenerations != int.MinValue)
{
generations = new FileGenerations();
}
String tmp = MaxGenerations == int.MinValue ? node.ReadStr("@file") : node.ReadStr("@file", null);
FileNameBase = engine.Xml.CombinePath(tmp == null ? "textOutput" : tmp);
fileName = FileNameBase;
rawTokens = node.ReadBool("@rawtokens", false);
Format = node.ReadStr("@format", null);
if (Format == null)
{
Format = node.ReadStr("format");
}
if (Format == "*")
{
Format = null;
}
Header = node.ReadStr("@header", null);
if (Header == null)
{
Header = node.ReadStr("header", null);
}
Footer = node.ReadStr("@footer", null);
if (Footer == null)
{
Footer = node.ReadStr("footer", null);
}
String cs = node.ReadStr("@encoding", null);
Encoding = cs == null ? Encoding.Default : Encoding.GetEncoding(cs);
//Predefine field orders if requested (implies linient mode)
fields = node.ReadStr("@fieldorder", null).SplitStandard();
if (fields != null && fields.Length > 0)
{
formatParms = new Object[fields.Length];
}
else
{
fields = null;
}
}
public JsonEndpoint(ImportEngine engine, XmlNode node)
: base(engine, node)
{
FileName = engine.Xml.CombinePath(node.ReadStr("@file"));
//pwOutputProvider = OutputStreamProviderBase.Create(null, node);
LineSeparator = node.ReadStrRaw("@linesep", _XmlRawMode.Trim | _XmlRawMode.DefaultOnNull, "\r\n");
Formatting = node.ReadBool("@formatted", false) ? Newtonsoft.Json.Formatting.Indented : Newtonsoft.Json.Formatting.None;
Root = node.ReadStr("@root", null);
Array = node.ReadBool("@array", Root != null);
if (Root != null && !Array)
{
throw new BMNodeException(node, "Cannot have array=[false] when root=[{0}].", Root);
}
}
public RunAdministrationSettings(ImportEngine engine, XmlNode node)
{
Engine = engine;
if (node == null)
{
FileName = null;
Capacity = DEF_CAPACITY;
}
else
{
FileName = node.ReadPath("@file", null);
Capacity = node.ReadInt("@capacity", DEF_CAPACITY);
Dump = node.ReadInt("@dump", 0);
}
}
protected Command(ImportEngine eng, XmlNode node)
{
WorkingDir = node.ReadStr("@curdir", null);
WorkingDir = eng.Xml.CombinePath(WorkingDir);
rawTokens = node.ReadBool("@rawtokens", false);
fields = node.ReadStr("@arguments", null).SplitStandard();
if (fields != null && fields.Length > 0)
{
formatParms = new Object[fields.Length];
}
else
{
fields = null;
}
}
public CopyCommand(ImportEngine eng, XmlNode node)
: base(eng, node)
{
overWrite = node.ReadBool("@overwrite", true);
src = node.ReadStr("@src");
dst = node.ReadStr("@dst", null);
dstDir = node.ReadStr("@dstdir", null);
if (dstDir == null && dst == null)
{
throw new BMNodeException(node, "Missing value for either @dst or @dstdir.");
}
if (dstDir != null && dst != null)
{
throw new BMNodeException(node, "Duplicate value: @dst and @dstdir are mutually exclusive.");
}
}
public DatasourceAdmin(PipelineContext ctx, XmlNode node)
: base(node)
{
Type = node.ReadStr("@type");
Active = node.ReadBool("@active", true);
LogAdds = node.ReadInt(1, "@logadds", -1);
MaxAdds = node.ReadInt(1, "@maxadds", -1);
MaxEmits = node.ReadInt(1, "@maxemits", -1);
String tmp = node.ReadStr(1, "@shiftlastruntime", null);
ShiftLastRuntime = computeRuntimeShift(tmp);
if (ShiftLastRuntime == int.MinValue)
{
throw new BMNodeException(node, "Invalid shiftlastruntime [{0}]: must be <int>[d|h|m|s].", tmp);
}
String pipelineName = node.ReadStr(1, "@pipeline", null);
Pipeline = ctx.ImportEngine.Pipelines.GetByNamesOrFirst(pipelineName, Name);
String endpoint = node.ReadStr(1, "@endpoint", null);
if (endpoint != null)
{
endpoint = endpoint.Replace("*", Name);
ctx.ImportEngine.Endpoints.CheckDataEndpoint(ctx, endpoint, true);
EndpointName = endpoint;
}
String endpointExpr = node.ReadStr(1, "@endpoint_expr", null);
if (endpointExpr != null)
{
if (endpoint != null)
{
throw new BMNodeException(node, "@endpoint and @endpoint_expr cannot be specified both.");
}
EndpointName = PerlRegex.Replace(endpointExpr, Name);
ctx.ImportEngine.Endpoints.CheckDataEndpoint(ctx, EndpointName, true);
}
Pipeline.CheckEndpoints(ctx, this);
//if (!Active) return; Zie notes: ws moet een datasource definitief kunnen worden uitgeschakeld. iets als active=true/false/disabled
Datasource = ImportEngine.CreateObject <Datasource> (Type);
Datasource.Init(ctx, node);
}
public PostProcessors(ImportEngine engine, XmlNode collNode)
{
postProcessors = new StringDict <IPostProcessor>();
if (collNode == null)
{
return;
}
var nodes = collNode.SelectNodes("postprocessor");
for (int i = 0; i < nodes.Count; i++)
{
XmlNode c = nodes[i];
IPostProcessor p = ImportEngine.CreateObject <IPostProcessor> (c, engine, c);
postProcessors.Add(p.Name, p);
}
}
public DelayedScriptHolder(ImportEngine engine, XmlNode node, String name)
{
this.Node = node;
String body = null;
if (node != null)
{
ScriptName = ImportEngine.ReadScriptNameOrBody(node, "@script", out body);
}
if (body != null)
{
ScriptName = ScriptExpressionHolder.GenerateScriptName("postprocessor_", name, node);
IsExpression = true;
engine.ScriptExpressions.AddExpression(ScriptName, body);
}
}
public Pipeline(ImportEngine engine, XmlNode node) : base(node)
{
ImportEngine = engine;
logger = engine.DebugLog.Clone("pipeline");
ScriptTypeName = node.ReadStr("@script", null);
DefaultConverters = node.ReadStr("@converters", null);
DefaultPostProcessors = node.ReadStr("@postprocessors", null);
DefaultEndpoint = node.ReadStr("@endpoint", null);
if (DefaultEndpoint == null)
{
if (engine.Endpoints.Count == 1)
{
DefaultEndpoint = engine.Endpoints[0].Name;
}
else if (engine.Endpoints.GetByName(Name, false) != null)
{
DefaultEndpoint = Name;
}
}
trace = node.ReadBool("@trace", false);
AdminCollection <PipelineAction> rawActions = new AdminCollection <PipelineAction>(node, "action", (x) => PipelineAction.Create(this, x), false);
definedActions = new List <ActionAdmin>();
for (int i = 0; i < rawActions.Count; i++)
{
var action = rawActions[i];
String[] keys = action.Name.SplitStandard();
for (int k = 0; k < keys.Length; k++)
{
definedActions.Add(new ActionAdmin(keys[k], i, action));
}
}
definedActions.Sort(cbSortAction);
var templNodes = node.SelectNodes("template");
templates = new List <PipelineTemplate>(templNodes.Count);
for (int i = 0; i < templNodes.Count; i++)
{
templates.Add(PipelineTemplate.Create(this, templNodes[i]));
}
Dump("");
}
public SortProcessor(ImportEngine engine, XmlNode node) : base(engine, node)
{
List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("sorter"), "key", false);
Sorter = JComparer.Create(list);
//Interpret undupper
XmlNode undupNode = node.SelectSingleNode("undupper");
if (undupNode != null)
{
Undupper = Sorter.Clone(undupNode.ReadStr("@from_sort", null));
if (Undupper == null)
{
list = KeyAndType.CreateKeyList(undupNode, "key", true);
Undupper = JComparer.Create(list);
}
XmlNode actionsNode = undupNode.SelectSingleNode("actions");
if (actionsNode != null)
{
undupActions = new UndupActions(engine, this, actionsNode);
}
}
//Interpret sort scripts
beforeSort = new DelayedScriptHolder(engine, node.SelectSingleNode("beforesort"), Name);
if (beforeSort.ScriptName == null)
{
beforeSort = null;
}
afterSort = new DelayedScriptHolder(engine, node.SelectSingleNode("aftersort"), Name);
if (afterSort.ScriptName == null)
{
afterSort = null;
}
}
public ESEndpoint(ImportEngine engine, XmlNode node)
: base(engine, node)
{
Connection = new ESConnection(node.ReadStr("@url"));
CacheSize = node.ReadInt("@cache", -1);
MaxParallel = node.ReadInt("@maxparallel", 0);
ReadOnly = node.ReadBool("@readonly", false);
XmlNode root = node.SelectSingleNode("indexes");
if (root == null)
{
root = node;
}
Indexes = new ESIndexDefinitions(engine.Xml, root, _loadConfig);
if (Indexes.Count == 0)
{
throw new BMNodeException(node, "At least 1 index+type is required!");
}
WaitFor = ESHealthCmd.SplitRequestedClusterStatus(node.ReadStr("waitfor/@status", "Green | Yellow"), out WaitForAlt);
WaitForTimeout = node.ReadInt("waitfor/@timeout", 30);
WaitForMustExcept = node.ReadBool("waitfor/@except", true);
}
private static Endpoint create (ImportEngine engine, XmlNode n)
{
String type = n.ReadStr("@type");
if (String.Equals("nop", type, StringComparison.InvariantCultureIgnoreCase)) return new Endpoint(engine, n);
return ImportEngine.CreateObject<Endpoint>(n, engine, n);
}
public Endpoint(ImportEngine engine, XmlNode node) : this(engine, node, 0) { }
public RepeatProcessor(ImportEngine engine, XmlNode node) : base(engine, node)
{
repeatCount = node.ReadInt("@repeat");
}
public ProcessHostCollection(ImportEngine engine, XmlNode collNode)
: base(collNode, "process", (n) => new ProcessHost(n), false)
{
logger = engine.ImportLog.Clone("processHost");
}
public PostProcessorBase(ImportEngine engine, XmlNode node)
{
name = node.ReadStr("@name");
instanceNo = -1;
}
public MoveCommand(ImportEngine eng, XmlNode node)
: base(eng, node)
{
}
