public static IUndupAction CreateAction(ImportEngine engine, IPostProcessor processor, XmlNode node) { String type = node.ReadStr("@type"); if ("add".Equals(type, StringComparison.OrdinalIgnoreCase)) { return(new UndupNumericAddAction(processor, node)); } if ("max".Equals(type, StringComparison.OrdinalIgnoreCase)) { return(new UndupNumericMaxAction(processor, node)); } if ("min".Equals(type, StringComparison.OrdinalIgnoreCase)) { return(new UndupNumericMinAction(processor, node)); } if ("mean".Equals(type, StringComparison.OrdinalIgnoreCase)) { return(new UndupNumericMeanAction(processor, node)); } if ("count".Equals(type, StringComparison.OrdinalIgnoreCase)) { return(new UndupCountAction(processor, node)); } if ("script".Equals(type, StringComparison.OrdinalIgnoreCase)) { return(new UndupScriptAction(engine, processor, node)); } throw new BMException("Unrecognized type [{0}] for a post process action.", type); //TODO make this more generic }
public MapReduceProcessor(ImportEngine engine, XmlNode node) : base(engine, node) { maxNullIndex = node.ReadInt("@max_null_index", -1); fanOut = node.ReadInt("@fan_out", 100); if (fanOut <= 0) { throw new BMNodeException(node, "Count should be > 0."); } if (node.ReadInt("write/@maxparallel", 1) > 0) { bufferSize = node.ReadInt("write/@buffer", 100); } readMaxParallel = node.ReadInt("read/@maxparallel", 1); directory = node.ReadStr("dir/@name", null); if (directory != null) { directory = engine.Xml.CombinePath(directory); keepFiles = node.ReadBool("dir/@keepfiles", false); compress = node.ReadBool("dir/@compress", true); } List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("sorter"), "key", false); sorter = JComparer.Create(list); XmlNode hashNode = node.SelectSingleNode("hasher"); if (hashNode == null) { hasher = sorter; } else { hasher = sorter.Clone(hashNode.ReadStr("@from_sort", null)); if (hasher == null) { list = KeyAndType.CreateKeyList(hashNode, "key", true); hasher = JComparer.Create(list); } } XmlNode undupNode = node.SelectSingleNode("undupper"); if (undupNode != null) { undupper = sorter.Clone(undupNode.ReadStr("@from_sort", null)); if (undupper == null) { list = KeyAndType.CreateKeyList(undupNode, "key", true); undupper = JComparer.Create(list); } XmlNode actionsNode = undupNode.SelectSingleNode("actions"); if (actionsNode != null) { undupActions = new UndupActions(engine, this, actionsNode); } } }
public PipelineContext(PipelineContext ctx, DatasourceAdmin ds, DatasourceReport report) { var eng = ctx.ImportEngine; Switches = ctx.Switches; NewLastUpdated = eng.StartTimeUtc; ImportEngine = eng; RunAdministrations = eng.RunAdministrations; DatasourceAdmin = ds; DatasourceReport = report; Pipeline = ds.Pipeline; ImportLog = eng.ImportLog.Clone(ds.Name); DebugLog = eng.DebugLog.Clone(ds.Name); ErrorLog = eng.ErrorLog.Clone(ds.Name); MissedLog = eng.MissedLog.Clone(ds.Name); ImportFlags = eng.ImportFlags; LogAdds = (ds.LogAdds > 0) ? ds.LogAdds : eng.LogAdds; MaxAdds = (ds.MaxAdds >= 0) ? ds.MaxAdds : eng.MaxAdds; MaxEmits = (ds.MaxEmits >= 0) ? ds.MaxEmits : eng.MaxEmits; if (MaxEmits < 0 && (ImportFlags & _ImportFlags.MaxAddsToMaxEmits) != 0) { MaxEmits = MaxAdds; } ImportLog.Log("Current maxAdds={0}, maxEmits={1}", MaxAdds, MaxEmits); }
public RunAdministrationSettings(ImportEngine engine, String fn, int cap, int dump) { Engine = engine; FileName = fn; Capacity = cap; Dump = dump; }
public Endpoint (ImportEngine engine, XmlNode node, ImportPipeline.ActiveMode defActiveMode=0) : base(node) { Engine = engine; if (defActiveMode == 0) defActiveMode = ImportPipeline.ActiveMode.Lazy | ImportPipeline.ActiveMode.Global; if ((defActiveMode & (ImportPipeline.ActiveMode.Local | ImportPipeline.ActiveMode.Global)) != 0) defActiveMode |= ImportPipeline.ActiveMode.Global; Flags = node.ReadEnum<DebugFlags>("@flags", 0); CloseMode = node.ReadEnum("@closemode", CloseMode.Normal); ActiveMode = node.ReadEnum("@active", defActiveMode); switch (ActiveMode) { case 0: ActiveMode = defActiveMode; break; case ImportPipeline.ActiveMode.False: case ImportPipeline.ActiveMode.True: break; default: if ((ActiveMode & (ImportPipeline.ActiveMode.False | ImportPipeline.ActiveMode.True)) != 0) throw new BMNodeException (node, "ActiveMode true/false cannot be combined with other flags. ActiveMode={0}.", ActiveMode); if ((ActiveMode & (ImportPipeline.ActiveMode.Local | ImportPipeline.ActiveMode.Global)) != 0) ActiveMode |= defActiveMode & (ImportPipeline.ActiveMode.Local | ImportPipeline.ActiveMode.Global); break; } }
public UndupScriptAction(ImportEngine engine, IPostProcessor processor, XmlNode node) { originalNode = node; ScriptName = node.ReadStr("@script", null); ScriptBody = node.ReadStr(null, null); if (ScriptBody == null) { if (ScriptName == null) { throw new BMNodeException(node, "@script should be specified if there is no script in the body."); } } else { ScriptBody = ScriptBody.TrimWhiteSpaceToNull(); if (ScriptBody != null && ScriptName != null) { throw new BMNodeException(node, "Cannot have a script in the body when @script is specified."); } var scriptHolder = engine.ScriptExpressions; bodyFunc = ScriptExpressionHolder.GenerateScriptName("postprocessor_undup", processor.Name, node); scriptHolder.AddUndupExpression(bodyFunc, ScriptBody); } }
public CommandEndpoint(ImportEngine engine, XmlNode node) : base(engine, node, ActiveMode.Lazy | ActiveMode.Local) { XmlNodeList chlds = node.ChildNodes;// node.SelectMandatoryNodes("*"); var cmds = new List <Command>(chlds.Count); for (int i = 0; i < chlds.Count; i++) { XmlNode n = chlds[i]; String name = n.Name; Command cmd; switch (name) { default: continue; case "command": case "exec": cmd = new ExecCommand(engine, n); break; case "delete": case "del": cmd = new DeleteCommand(engine, n); break; case "copy": cmd = new CopyCommand(engine, n); break; case "move": cmd = new MoveCommand(engine, n); break; } cmds.Add(cmd); } Commands = cmds.ToArray(); if (cmds.Count == 0) { engine.ImportLog.Log("Endpoint [{0}] has no commands. Resulting in a no-op.", Name); } }
public TopProcessor(ImportEngine engine, XmlNode node) : base(engine, node) { List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("sorter"), "key", true); sorter = JComparer.Create(list); topCount = node.ReadInt("@count"); reverse = node.ReadBool("@reverse", false); sortAfter = node.ReadEnum("@sortafter", _SortAfter.True); }
public Endpoints(ImportEngine engine, XmlNode collNode) : base(collNode, "endpoint", (n) => create (engine, n), false) { this.engine = engine; if (GetByName("nop", false) == null) { Add (new Endpoint("nop")); //always have a NOP endpoint for free } }
//private ImportEngine engine; //private SortProcessor sortProcessor; //private XmlNode actionsNode; public UndupActions(ImportEngine engine, IPostProcessor processor, XmlNode node) { XmlNodeList nodes = node.SelectNodes("action"); actions = new List <IUndupAction>(nodes.Count); foreach (XmlNode child in nodes) { actions.Add(CreateAction(engine, processor, child)); } }
public PipelineContext(ImportEngine eng) { ImportEngine = eng; ImportLog = eng.ImportLog; DebugLog = eng.DebugLog; ErrorLog = eng.ErrorLog; MissedLog = eng.MissedLog; ImportFlags = eng.ImportFlags; NewLastUpdated = eng.StartTimeUtc; RunAdministrations = eng.RunAdministrations; Switches = new Switches(eng.SwitchesFromXml + " " + eng.Switches); //Switches are overriding }
public ExecCommand(ImportEngine eng, XmlNode node) : base(eng, node) { cmd = node.ReadStr("@cmd"); if (!cmd.StartsWith("cmd ")) { cmd = "cmd /c " + cmd; } settings = new ProcessHostSettings(); settings.LogName = node.ReadStr("@log", settings.LogName); settings.LogName = node.ReadStr("@errorlog", settings.LogName); settings.ExeName = "cmd.exe"; }
public CsvEndpoint(ImportEngine engine, XmlNode node) : base(engine, node, ActiveMode.Lazy | ActiveMode.Local) { MaxGenerations = node.ReadInt("@generations", int.MinValue); if (MaxGenerations != int.MinValue) { generations = new FileGenerations(); } String tmp = MaxGenerations == int.MinValue ? node.ReadStr("@file") : node.ReadStr("@file", null); FileNameBase = engine.Xml.CombinePath(tmp == null ? "csvOutput" : tmp); fileName = FileNameBase; trim = node.ReadBool("@trim", true); lenient = node.ReadBool("@lenient", false); delimChar = CsvDatasource.readChar(node, "@dlm", ','); quoteChar = CsvDatasource.readChar(node, "@quote", '"'); commentChar = CsvDatasource.readChar(node, "@comment", '#'); //Predefine field orders if requested (implies linient mode) String fields = node.ReadStr("@fields", null); String fieldsOrder = node.ReadStr("@fieldorder", null); if (fields != null && fieldsOrder != null) { throw new BMNodeException(node, "Attributes [fields] and [fieldorder] cannot be specified together."); } String[] order; if (fields != null) { order = fields.SplitStandard(); } else if (fieldsOrder != null) { order = fieldsOrder.SplitStandard(); } else { order = null; } if (order != null) { lenient = true; foreach (var fld in order) { keyToIndex(fld); } selectiveExport = fields != null; } }
public TextEndpoint(ImportEngine engine, XmlNode node) : base(engine, node, ActiveMode.Lazy | ActiveMode.Local) { MaxGenerations = node.ReadInt("@generations", int.MinValue); if (MaxGenerations != int.MinValue) { generations = new FileGenerations(); } String tmp = MaxGenerations == int.MinValue ? node.ReadStr("@file") : node.ReadStr("@file", null); FileNameBase = engine.Xml.CombinePath(tmp == null ? "textOutput" : tmp); fileName = FileNameBase; rawTokens = node.ReadBool("@rawtokens", false); Format = node.ReadStr("@format", null); if (Format == null) { Format = node.ReadStr("format"); } if (Format == "*") { Format = null; } Header = node.ReadStr("@header", null); if (Header == null) { Header = node.ReadStr("header", null); } Footer = node.ReadStr("@footer", null); if (Footer == null) { Footer = node.ReadStr("footer", null); } String cs = node.ReadStr("@encoding", null); Encoding = cs == null ? Encoding.Default : Encoding.GetEncoding(cs); //Predefine field orders if requested (implies linient mode) fields = node.ReadStr("@fieldorder", null).SplitStandard(); if (fields != null && fields.Length > 0) { formatParms = new Object[fields.Length]; } else { fields = null; } }
public JsonEndpoint(ImportEngine engine, XmlNode node) : base(engine, node) { FileName = engine.Xml.CombinePath(node.ReadStr("@file")); //pwOutputProvider = OutputStreamProviderBase.Create(null, node); LineSeparator = node.ReadStrRaw("@linesep", _XmlRawMode.Trim | _XmlRawMode.DefaultOnNull, "\r\n"); Formatting = node.ReadBool("@formatted", false) ? Newtonsoft.Json.Formatting.Indented : Newtonsoft.Json.Formatting.None; Root = node.ReadStr("@root", null); Array = node.ReadBool("@array", Root != null); if (Root != null && !Array) { throw new BMNodeException(node, "Cannot have array=[false] when root=[{0}].", Root); } }
public RunAdministrationSettings(ImportEngine engine, XmlNode node) { Engine = engine; if (node == null) { FileName = null; Capacity = DEF_CAPACITY; } else { FileName = node.ReadPath("@file", null); Capacity = node.ReadInt("@capacity", DEF_CAPACITY); Dump = node.ReadInt("@dump", 0); } }
protected Command(ImportEngine eng, XmlNode node) { WorkingDir = node.ReadStr("@curdir", null); WorkingDir = eng.Xml.CombinePath(WorkingDir); rawTokens = node.ReadBool("@rawtokens", false); fields = node.ReadStr("@arguments", null).SplitStandard(); if (fields != null && fields.Length > 0) { formatParms = new Object[fields.Length]; } else { fields = null; } }
public CopyCommand(ImportEngine eng, XmlNode node) : base(eng, node) { overWrite = node.ReadBool("@overwrite", true); src = node.ReadStr("@src"); dst = node.ReadStr("@dst", null); dstDir = node.ReadStr("@dstdir", null); if (dstDir == null && dst == null) { throw new BMNodeException(node, "Missing value for either @dst or @dstdir."); } if (dstDir != null && dst != null) { throw new BMNodeException(node, "Duplicate value: @dst and @dstdir are mutually exclusive."); } }
public DatasourceAdmin(PipelineContext ctx, XmlNode node) : base(node) { Type = node.ReadStr("@type"); Active = node.ReadBool("@active", true); LogAdds = node.ReadInt(1, "@logadds", -1); MaxAdds = node.ReadInt(1, "@maxadds", -1); MaxEmits = node.ReadInt(1, "@maxemits", -1); String tmp = node.ReadStr(1, "@shiftlastruntime", null); ShiftLastRuntime = computeRuntimeShift(tmp); if (ShiftLastRuntime == int.MinValue) { throw new BMNodeException(node, "Invalid shiftlastruntime [{0}]: must be <int>[d|h|m|s].", tmp); } String pipelineName = node.ReadStr(1, "@pipeline", null); Pipeline = ctx.ImportEngine.Pipelines.GetByNamesOrFirst(pipelineName, Name); String endpoint = node.ReadStr(1, "@endpoint", null); if (endpoint != null) { endpoint = endpoint.Replace("*", Name); ctx.ImportEngine.Endpoints.CheckDataEndpoint(ctx, endpoint, true); EndpointName = endpoint; } String endpointExpr = node.ReadStr(1, "@endpoint_expr", null); if (endpointExpr != null) { if (endpoint != null) { throw new BMNodeException(node, "@endpoint and @endpoint_expr cannot be specified both."); } EndpointName = PerlRegex.Replace(endpointExpr, Name); ctx.ImportEngine.Endpoints.CheckDataEndpoint(ctx, EndpointName, true); } Pipeline.CheckEndpoints(ctx, this); //if (!Active) return; Zie notes: ws moet een datasource definitief kunnen worden uitgeschakeld. iets als active=true/false/disabled Datasource = ImportEngine.CreateObject <Datasource> (Type); Datasource.Init(ctx, node); }
public PostProcessors(ImportEngine engine, XmlNode collNode) { postProcessors = new StringDict <IPostProcessor>(); if (collNode == null) { return; } var nodes = collNode.SelectNodes("postprocessor"); for (int i = 0; i < nodes.Count; i++) { XmlNode c = nodes[i]; IPostProcessor p = ImportEngine.CreateObject <IPostProcessor> (c, engine, c); postProcessors.Add(p.Name, p); } }
public DelayedScriptHolder(ImportEngine engine, XmlNode node, String name) { this.Node = node; String body = null; if (node != null) { ScriptName = ImportEngine.ReadScriptNameOrBody(node, "@script", out body); } if (body != null) { ScriptName = ScriptExpressionHolder.GenerateScriptName("postprocessor_", name, node); IsExpression = true; engine.ScriptExpressions.AddExpression(ScriptName, body); } }
public Pipeline(ImportEngine engine, XmlNode node) : base(node) { ImportEngine = engine; logger = engine.DebugLog.Clone("pipeline"); ScriptTypeName = node.ReadStr("@script", null); DefaultConverters = node.ReadStr("@converters", null); DefaultPostProcessors = node.ReadStr("@postprocessors", null); DefaultEndpoint = node.ReadStr("@endpoint", null); if (DefaultEndpoint == null) { if (engine.Endpoints.Count == 1) { DefaultEndpoint = engine.Endpoints[0].Name; } else if (engine.Endpoints.GetByName(Name, false) != null) { DefaultEndpoint = Name; } } trace = node.ReadBool("@trace", false); AdminCollection <PipelineAction> rawActions = new AdminCollection <PipelineAction>(node, "action", (x) => PipelineAction.Create(this, x), false); definedActions = new List <ActionAdmin>(); for (int i = 0; i < rawActions.Count; i++) { var action = rawActions[i]; String[] keys = action.Name.SplitStandard(); for (int k = 0; k < keys.Length; k++) { definedActions.Add(new ActionAdmin(keys[k], i, action)); } } definedActions.Sort(cbSortAction); var templNodes = node.SelectNodes("template"); templates = new List <PipelineTemplate>(templNodes.Count); for (int i = 0; i < templNodes.Count; i++) { templates.Add(PipelineTemplate.Create(this, templNodes[i])); } Dump(""); }
public SortProcessor(ImportEngine engine, XmlNode node) : base(engine, node) { List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("sorter"), "key", false); Sorter = JComparer.Create(list); //Interpret undupper XmlNode undupNode = node.SelectSingleNode("undupper"); if (undupNode != null) { Undupper = Sorter.Clone(undupNode.ReadStr("@from_sort", null)); if (Undupper == null) { list = KeyAndType.CreateKeyList(undupNode, "key", true); Undupper = JComparer.Create(list); } XmlNode actionsNode = undupNode.SelectSingleNode("actions"); if (actionsNode != null) { undupActions = new UndupActions(engine, this, actionsNode); } } //Interpret sort scripts beforeSort = new DelayedScriptHolder(engine, node.SelectSingleNode("beforesort"), Name); if (beforeSort.ScriptName == null) { beforeSort = null; } afterSort = new DelayedScriptHolder(engine, node.SelectSingleNode("aftersort"), Name); if (afterSort.ScriptName == null) { afterSort = null; } }
public ESEndpoint(ImportEngine engine, XmlNode node) : base(engine, node) { Connection = new ESConnection(node.ReadStr("@url")); CacheSize = node.ReadInt("@cache", -1); MaxParallel = node.ReadInt("@maxparallel", 0); ReadOnly = node.ReadBool("@readonly", false); XmlNode root = node.SelectSingleNode("indexes"); if (root == null) { root = node; } Indexes = new ESIndexDefinitions(engine.Xml, root, _loadConfig); if (Indexes.Count == 0) { throw new BMNodeException(node, "At least 1 index+type is required!"); } WaitFor = ESHealthCmd.SplitRequestedClusterStatus(node.ReadStr("waitfor/@status", "Green | Yellow"), out WaitForAlt); WaitForTimeout = node.ReadInt("waitfor/@timeout", 30); WaitForMustExcept = node.ReadBool("waitfor/@except", true); }
private static Endpoint create (ImportEngine engine, XmlNode n) { String type = n.ReadStr("@type"); if (String.Equals("nop", type, StringComparison.InvariantCultureIgnoreCase)) return new Endpoint(engine, n); return ImportEngine.CreateObject<Endpoint>(n, engine, n); }
public Endpoint(ImportEngine engine, XmlNode node) : this(engine, node, 0) { }
public RepeatProcessor(ImportEngine engine, XmlNode node) : base(engine, node) { repeatCount = node.ReadInt("@repeat"); }
public ProcessHostCollection(ImportEngine engine, XmlNode collNode) : base(collNode, "process", (n) => new ProcessHost(n), false) { logger = engine.ImportLog.Clone("processHost"); }
public PostProcessorBase(ImportEngine engine, XmlNode node) { name = node.ReadStr("@name"); instanceNo = -1; }
public MoveCommand(ImportEngine eng, XmlNode node) : base(eng, node) { }