//Mainly for testing public RunAdministration(String ds, DateTime dt, _ImportFlags flags, int added = 0) { RunDateUtc = dt.ToUniversalTime(); DataSource = ds; ImportFlags = flags; Added = added; }
public PipelineContext(PipelineContext ctx, DatasourceAdmin ds, DatasourceReport report) { var eng = ctx.ImportEngine; Switches = ctx.Switches; NewLastUpdated = eng.StartTimeUtc; ImportEngine = eng; RunAdministrations = eng.RunAdministrations; DatasourceAdmin = ds; DatasourceReport = report; Pipeline = ds.Pipeline; ImportLog = eng.ImportLog.Clone(ds.Name); DebugLog = eng.DebugLog.Clone(ds.Name); ErrorLog = eng.ErrorLog.Clone(ds.Name); MissedLog = eng.MissedLog.Clone(ds.Name); ImportFlags = eng.ImportFlags; LogAdds = (ds.LogAdds > 0) ? ds.LogAdds : eng.LogAdds; MaxAdds = (ds.MaxAdds >= 0) ? ds.MaxAdds : eng.MaxAdds; MaxEmits = (ds.MaxEmits >= 0) ? ds.MaxEmits : eng.MaxEmits; if (MaxEmits < 0 && (ImportFlags & _ImportFlags.MaxAddsToMaxEmits) != 0) { MaxEmits = MaxAdds; } ImportLog.Log("Current maxAdds={0}, maxEmits={1}", MaxAdds, MaxEmits); }
private XmlHelper loadXml(String fileName, _ImportFlags flags = _ImportFlags.UseFlagsFromXml) { XmlHelper tmp = new XmlHelper(fileName); if (TemplateFactory == null) { String factoryClass = tmp.ReadStr("@templatefactory", typeof(TemplateFactory).FullName); TemplateFactory = CreateObject <ITemplateFactory>(factoryClass, this, tmp); } _ImportFlags flagsFromXml = tmp.ReadEnum("@importflags", (_ImportFlags)0); if (((ImportFlags | flagsFromXml) & _ImportFlags.DebugTemplate) != 0) { TemplateFactory.AutoWriteGenerated = true; } ImportLog.Log("Flags before load={0}", ImportFlags); XmlHelper xml = new XmlHelper(); ITemplateEngine eng = TemplateFactory.CreateEngine(); eng.Variables.Set("IMPORT_ROOT", Path.GetDirectoryName(tmp.FileName)); tmp = null; eng.LoadFromFile(fileName); MainVariables = eng.Variables; FileVariables = eng.FileVariables; TemplateFactory.InitialVariables = FileVariables; xml.Load(eng.ResultAsReader(), fileName); return(xml); }
public RunAdministration(JObject obj) { RunDateUtc = obj.ReadDate(ADM_DATE); DataSource = obj.ReadStr(ADM_DS); State = Invariant.ToEnum(obj.ReadStr(ADM_STATE), _ErrorState.Error); ImportFlags = Invariant.ToEnum(obj.ReadStr(ADM_FLAGS), (_ImportFlags)0); Added = obj.ReadInt(ADM_ADDED); Deleted = obj.ReadInt(ADM_DELETED); Skipped = obj.ReadInt(ADM_SKIPPED); Emitted = obj.ReadInt(ADM_EMITTED); Errors = obj.ReadInt(ADM_ERRORS); }
public RunAdministration(PipelineContext ctx) { RunDateUtc = ctx.NewLastUpdated; DataSource = ctx.DatasourceAdmin.Name; State = ctx.ErrorState; Added = ctx.Added; Deleted = ctx.Deleted; Skipped = ctx.Skipped; Emitted = ctx.Emitted; Errors = ctx.Errors; ImportFlags = ctx.ImportFlags; }
RunAdministration addRun(RunAdministrations list, String ds, int days, _ImportFlags flags = (_ImportFlags)0, int added = 0) { var obj = new RunAdministration(ds, DateTime.Now.AddDays(days), flags, added); list.Add(obj); Console.WriteLine(); foreach (var a in list) { Console.WriteLine("-- " + a); } return(obj); }
public PipelineContext(ImportEngine eng) { ImportEngine = eng; ImportLog = eng.ImportLog; DebugLog = eng.DebugLog; ErrorLog = eng.ErrorLog; MissedLog = eng.MissedLog; ImportFlags = eng.ImportFlags; NewLastUpdated = eng.StartTimeUtc; RunAdministrations = eng.RunAdministrations; Switches = new Switches(eng.SwitchesFromXml + " " + eng.Switches); //Switches are overriding }
private void uiFromFlags(ImportEngine eng) { foreach (var c in grpFlags.Controls) { CheckBox cb = c as CheckBox; if (cb == null) { continue; } _ImportFlags flag = Invariant.ToEnum <_ImportFlags>(cb.Text); cb.Checked = (eng.ImportFlags & flag) != 0; } }
public void Start(_ImportFlags flags, String switches, String xml, String[] activeDS, int maxRecords, int maxEmits) { domain = AppDomain.CreateDomain("import"); Type type = typeof(EngineWrapper); EngineWrapper wrapper = (EngineWrapper)domain.CreateInstanceAndUnwrap(type.Assembly.FullName, type.FullName, false, BindingFlags.CreateInstance, null, null, Invariant.Culture, null); action = wrapper.Run; asyncResult = action.BeginInvoke(flags, switches, xml, activeDS, maxRecords, maxEmits, null, null); started = true; return; }
private _ImportFlags uiToFlags() { _ImportFlags flags = 0; foreach (var c in grpFlags.Controls) { CheckBox cb = c as CheckBox; if (cb == null || !cb.Checked) { continue; } flags |= Invariant.ToEnum <_ImportFlags>(cb.Text); } return(flags); }
public ImportReport Run(_ImportFlags flags, String switches, String xml, String[] activeDS, int maxAdds, int maxEmits) { try { using (ImportEngine engine = new ImportEngine()) { engine.Switches = switches; engine.ImportFlags = flags; engine.Load(xml); engine.MaxAdds = maxAdds; engine.MaxEmits = maxEmits; return(engine.Import(activeDS)); } } catch (Exception e) { Logs.ErrorLog.Log(e); throw new Exception(e.Message); } }
public void IncrementEmitted() { if (MaxEmits >= 0 && Emitted >= MaxEmits) { ImportLog.Log("MAX EMITS EXCEEDED, {0}", GetStats()); throw Exceeded = new MaxAddsExceededException(Emitted, "emits"); } switch ((++Emitted) % LogAdds) { case 0: if ((this.ImportFlags & _ImportFlags.LogEmits) == 0) { if (Added != 0) { break; } } ImportFlags |= _ImportFlags.LogEmits; ImportLog.Log(_LogType.ltTimer, "Emitted {0} records", Emitted); break; } }
private static int runAsConsole(String[] args) { try { bool b = Bitmanager.Core.ConsoleHelpers.AttachConsole(); if (!b) { b = Bitmanager.Core.ConsoleHelpers.AllocConsole(); } } catch { } try { Thread.CurrentThread.CurrentCulture = System.Globalization.CultureInfo.InvariantCulture; var cmd = new CommandLineParms(args); if (cmd.NamedArgs.ContainsKey("?") || cmd.NamedArgs.ContainsKey("help")) { goto WRITE_SYNTAX; } String responseFile = cmd.NamedArgs.OptGetItem("resp"); if (responseFile != null) { if (cmd.Args.Count != 0) { goto WRITE_SYNTAX_ERR; } cmd = new CommandLineParms(responseFile); } _ImportFlags flags = Invariant.ToEnum <_ImportFlags>(cmd.NamedArgs.OptGetItem("flags"), _ImportFlags.UseFlagsFromXml); int maxAdds = Invariant.ToInt32(cmd.NamedArgs.OptGetItem("maxadds"), -1); int maxEmits = Invariant.ToInt32(cmd.NamedArgs.OptGetItem("maxemits"), -1); if (cmd.Args.Count == 0) { goto WRITE_SYNTAX_ERR; } using (ImportEngine eng = new ImportEngine()) { eng.MaxAdds = maxAdds; eng.MaxEmits = maxEmits; eng.ImportFlags = flags; String[] dsList = new String[cmd.Args.Count - 1]; for (int i = 1; i < cmd.Args.Count; i++) { dsList[i - 1] = cmd.Args[i]; } eng.Load(cmd.Args[0]); eng.Import(dsList.Length == 0 ? null : dsList); } return(0); WRITE_SYNTAX_ERR: logError("Invalid commandline: {0}", Environment.CommandLine); WRITE_SYNTAX: logError(""); logError("Syntax: <importxml file> [list of datasources] [/flags:<importflags>] [/maxadds:<number>] [/maxemits:<number>] [/$$xxxx$$:<value>"); logError(" or: /resp:<responsefile> with 1 option per line"); return(12); } catch (Exception e) { logError("Error: {0}\r\nType: {1}\r\nStack:\r\n{2}", e.Message, e.GetType().FullName, e.StackTrace); return(12); } }
public void Load(XmlHelper xml) { Xml = xml; String dir = xml.FileName; if (!String.IsNullOrEmpty(dir)) { dir = Path.GetDirectoryName(xml.FileName); } Environment.SetEnvironmentVariable("IMPORT_ROOT", dir); fillTikaVars(); _ImportFlags flags = ImportFlags; ImportFlags = xml.ReadEnum("@importflags", (_ImportFlags)0); if ((flags & _ImportFlags.UseFlagsFromXml) == 0) { ImportFlags = flags; } LogAdds = xml.ReadInt("@logadds", LogAdds); MaxAdds = xml.ReadInt("@maxadds", MaxAdds); SwitchesFromXml = xml.ReadStr("@switches", null); ImportLog.Log("Loading import xml: flags={0}, logadds={1}, maxadds={2}, file={3}.", ImportFlags, LogAdds, MaxAdds, xml.FileName); binDir = Xml.CombinePath(Xml.ReadStr("@bindir", "bin")); if (Directory.Exists(binDir)) { ImportLog.Log(_LogType.ltInfo, "Using extra bin dir: {0}", binDir); } else { binDir = null; ImportLog.Log(_LogType.ltInfo, "No bin dir found... All executables are loaded from {0}.\r\nCheck bin dir={1}.", Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), binDir); } TempDir = IOUtils.AddSlash(Xml.CombinePath("temp")); IOUtils.ForceDirectories(TempDir, true); XmlNode x = Xml.SelectSingleNode("report"); if (x == null) { Reporter = null; } else { Reporter = new MailReporter(x); } ImportLog.Log("loaded reporter: {0}", Reporter); RunAdminSettings = new RunAdministrationSettings(this, Xml.SelectSingleNode("runadmin")); ImportLog.Log("loaded runadmin settings: {0}", RunAdminSettings); //Load the supplied script ImportLog.Log(_LogType.ltTimerStart, "loading: scripts"); XmlNode scriptNode = xml.SelectSingleNode("script"); //Create the holder for the expressions ScriptExpressions = new ScriptExpressionHolder(); ImportLog.Log(_LogType.ltTimer, "loading: helper process definitions "); ProcessHostCollection = new ProcessHostCollection(this, xml.SelectSingleNode("processes")); ImportLog.Log(_LogType.ltTimer, "loading: endpoints"); Endpoints = new Endpoints(this, xml.SelectMandatoryNode("endpoints")); ImportLog.Log(_LogType.ltTimer, "loading: post-processors"); PostProcessors = new PostProcessors(this, xml.SelectSingleNode("postprocessors")); ImportLog.Log(_LogType.ltTimer, "loading: converters"); Converters = new Converters( xml.SelectSingleNode("converters"), "converter", (node) => Converter.Create(node), false); ImportLog.Log(_LogType.ltTimer, "loading: categories"); Categories = new NamedAdminCollection <CategoryCollection>( xml.SelectSingleNode("categories"), "collection", (node) => new CategoryCollection(node), true); ImportLog.Log(_LogType.ltTimer, "loading: pipelines"); Pipelines = new NamedAdminCollection <Pipeline>( xml.SelectMandatoryNode("pipelines"), "pipeline", (node) => new Pipeline(this, node), true); ImportLog.Log(_LogType.ltTimer, "loading: datasources"); PipelineContext ctx = new PipelineContext(this); Datasources = new NamedAdminCollection <DatasourceAdmin>( xml.SelectMandatoryNode("datasources"), "datasource", (node) => new DatasourceAdmin(ctx, node), true); //Compile script if needed if (ScriptExpressions.Count > 0 || scriptNode != null) { ScriptHost = new ScriptHost(ScriptHostFlags.Default, TemplateFactory); if (scriptNode != null) { String fn = scriptNode.ReadStr("@file", null); if (fn != null) { ScriptHost.AddFile(xml.CombinePath(fn)); } } ScriptHost.ExtraSearchPath = binDir; if (ScriptExpressions.Count > 0) { String fn = TempDir + "_ScriptExpressions.cs"; ScriptExpressions.SaveAndClose(fn); ScriptHost.AddFile(fn); } ScriptHost.AddReference(Assembly.GetExecutingAssembly()); ScriptHost.AddReference(typeof(Bitmanager.Json.JsonExt)); ScriptHost.AddReference(typeof(Bitmanager.Elastic.ESConnection)); ScriptHost.Compile(); } ImportLog.Log(_LogType.ltTimerStop, "loading: finished. Loaded {0} datasources, {1} pipelines, {2} endpoints, {3} converters, {4} category collections.", Datasources.Count, Pipelines.Count, Endpoints.Count, Converters.Count, Categories.Count); }