//Mainly for testing
public RunAdministration(String ds, DateTime dt, _ImportFlags flags, int added = 0)
{
RunDateUtc = dt.ToUniversalTime();
DataSource = ds;
ImportFlags = flags;
Added = added;
}
public PipelineContext(PipelineContext ctx, DatasourceAdmin ds, DatasourceReport report)
{
var eng = ctx.ImportEngine;
Switches = ctx.Switches;
NewLastUpdated = eng.StartTimeUtc;
ImportEngine = eng;
RunAdministrations = eng.RunAdministrations;
DatasourceAdmin = ds;
DatasourceReport = report;
Pipeline = ds.Pipeline;
ImportLog = eng.ImportLog.Clone(ds.Name);
DebugLog = eng.DebugLog.Clone(ds.Name);
ErrorLog = eng.ErrorLog.Clone(ds.Name);
MissedLog = eng.MissedLog.Clone(ds.Name);
ImportFlags = eng.ImportFlags;
LogAdds = (ds.LogAdds > 0) ? ds.LogAdds : eng.LogAdds;
MaxAdds = (ds.MaxAdds >= 0) ? ds.MaxAdds : eng.MaxAdds;
MaxEmits = (ds.MaxEmits >= 0) ? ds.MaxEmits : eng.MaxEmits;
if (MaxEmits < 0 && (ImportFlags & _ImportFlags.MaxAddsToMaxEmits) != 0)
{
MaxEmits = MaxAdds;
}
ImportLog.Log("Current maxAdds={0}, maxEmits={1}", MaxAdds, MaxEmits);
}
private XmlHelper loadXml(String fileName, _ImportFlags flags = _ImportFlags.UseFlagsFromXml)
{
XmlHelper tmp = new XmlHelper(fileName);
if (TemplateFactory == null)
{
String factoryClass = tmp.ReadStr("@templatefactory", typeof(TemplateFactory).FullName);
TemplateFactory = CreateObject <ITemplateFactory>(factoryClass, this, tmp);
}
_ImportFlags flagsFromXml = tmp.ReadEnum("@importflags", (_ImportFlags)0);
if (((ImportFlags | flagsFromXml) & _ImportFlags.DebugTemplate) != 0)
{
TemplateFactory.AutoWriteGenerated = true;
}
ImportLog.Log("Flags before load={0}", ImportFlags);
XmlHelper xml = new XmlHelper();
ITemplateEngine eng = TemplateFactory.CreateEngine();
eng.Variables.Set("IMPORT_ROOT", Path.GetDirectoryName(tmp.FileName));
tmp = null;
eng.LoadFromFile(fileName);
MainVariables = eng.Variables;
FileVariables = eng.FileVariables;
TemplateFactory.InitialVariables = FileVariables;
xml.Load(eng.ResultAsReader(), fileName);
return(xml);
}
public RunAdministration(JObject obj)
{
RunDateUtc = obj.ReadDate(ADM_DATE);
DataSource = obj.ReadStr(ADM_DS);
State = Invariant.ToEnum(obj.ReadStr(ADM_STATE), _ErrorState.Error);
ImportFlags = Invariant.ToEnum(obj.ReadStr(ADM_FLAGS), (_ImportFlags)0);
Added = obj.ReadInt(ADM_ADDED);
Deleted = obj.ReadInt(ADM_DELETED);
Skipped = obj.ReadInt(ADM_SKIPPED);
Emitted = obj.ReadInt(ADM_EMITTED);
Errors = obj.ReadInt(ADM_ERRORS);
}
public RunAdministration(PipelineContext ctx)
{
RunDateUtc = ctx.NewLastUpdated;
DataSource = ctx.DatasourceAdmin.Name;
State = ctx.ErrorState;
Added = ctx.Added;
Deleted = ctx.Deleted;
Skipped = ctx.Skipped;
Emitted = ctx.Emitted;
Errors = ctx.Errors;
ImportFlags = ctx.ImportFlags;
}
RunAdministration addRun(RunAdministrations list, String ds, int days, _ImportFlags flags = (_ImportFlags)0, int added = 0)
{
var obj = new RunAdministration(ds, DateTime.Now.AddDays(days), flags, added);
list.Add(obj);
Console.WriteLine();
foreach (var a in list)
{
Console.WriteLine("-- " + a);
}
return(obj);
}
public PipelineContext(ImportEngine eng)
{
ImportEngine = eng;
ImportLog = eng.ImportLog;
DebugLog = eng.DebugLog;
ErrorLog = eng.ErrorLog;
MissedLog = eng.MissedLog;
ImportFlags = eng.ImportFlags;
NewLastUpdated = eng.StartTimeUtc;
RunAdministrations = eng.RunAdministrations;
Switches = new Switches(eng.SwitchesFromXml + " " + eng.Switches); //Switches are overriding
}
private void uiFromFlags(ImportEngine eng)
{
foreach (var c in grpFlags.Controls)
{
CheckBox cb = c as CheckBox;
if (cb == null)
{
continue;
}
_ImportFlags flag = Invariant.ToEnum <_ImportFlags>(cb.Text);
cb.Checked = (eng.ImportFlags & flag) != 0;
}
}
public void Start(_ImportFlags flags, String switches, String xml, String[] activeDS, int maxRecords, int maxEmits)
{
domain = AppDomain.CreateDomain("import");
Type type = typeof(EngineWrapper);
EngineWrapper wrapper = (EngineWrapper)domain.CreateInstanceAndUnwrap(type.Assembly.FullName, type.FullName, false, BindingFlags.CreateInstance, null, null, Invariant.Culture, null);
action = wrapper.Run;
asyncResult = action.BeginInvoke(flags, switches, xml, activeDS, maxRecords, maxEmits, null, null);
started = true;
return;
}
private _ImportFlags uiToFlags()
{
_ImportFlags flags = 0;
foreach (var c in grpFlags.Controls)
{
CheckBox cb = c as CheckBox;
if (cb == null || !cb.Checked)
{
continue;
}
flags |= Invariant.ToEnum <_ImportFlags>(cb.Text);
}
return(flags);
}
public ImportReport Run(_ImportFlags flags, String switches, String xml, String[] activeDS, int maxAdds, int maxEmits)
{
try
{
using (ImportEngine engine = new ImportEngine())
{
engine.Switches = switches;
engine.ImportFlags = flags;
engine.Load(xml);
engine.MaxAdds = maxAdds;
engine.MaxEmits = maxEmits;
return(engine.Import(activeDS));
}
}
catch (Exception e)
{
Logs.ErrorLog.Log(e);
throw new Exception(e.Message);
}
}
public void IncrementEmitted()
{
if (MaxEmits >= 0 && Emitted >= MaxEmits)
{
ImportLog.Log("MAX EMITS EXCEEDED, {0}", GetStats());
throw Exceeded = new MaxAddsExceededException(Emitted, "emits");
}
switch ((++Emitted) % LogAdds)
{
case 0:
if ((this.ImportFlags & _ImportFlags.LogEmits) == 0)
{
if (Added != 0)
{
break;
}
}
ImportFlags |= _ImportFlags.LogEmits;
ImportLog.Log(_LogType.ltTimer, "Emitted {0} records", Emitted); break;
}
}
private static int runAsConsole(String[] args)
{
try
{
bool b = Bitmanager.Core.ConsoleHelpers.AttachConsole();
if (!b)
{
b = Bitmanager.Core.ConsoleHelpers.AllocConsole();
}
}
catch { }
try
{
Thread.CurrentThread.CurrentCulture = System.Globalization.CultureInfo.InvariantCulture;
var cmd = new CommandLineParms(args);
if (cmd.NamedArgs.ContainsKey("?") || cmd.NamedArgs.ContainsKey("help"))
{
goto WRITE_SYNTAX;
}
String responseFile = cmd.NamedArgs.OptGetItem("resp");
if (responseFile != null)
{
if (cmd.Args.Count != 0)
{
goto WRITE_SYNTAX_ERR;
}
cmd = new CommandLineParms(responseFile);
}
_ImportFlags flags = Invariant.ToEnum <_ImportFlags>(cmd.NamedArgs.OptGetItem("flags"), _ImportFlags.UseFlagsFromXml);
int maxAdds = Invariant.ToInt32(cmd.NamedArgs.OptGetItem("maxadds"), -1);
int maxEmits = Invariant.ToInt32(cmd.NamedArgs.OptGetItem("maxemits"), -1);
if (cmd.Args.Count == 0)
{
goto WRITE_SYNTAX_ERR;
}
using (ImportEngine eng = new ImportEngine())
{
eng.MaxAdds = maxAdds;
eng.MaxEmits = maxEmits;
eng.ImportFlags = flags;
String[] dsList = new String[cmd.Args.Count - 1];
for (int i = 1; i < cmd.Args.Count; i++)
{
dsList[i - 1] = cmd.Args[i];
}
eng.Load(cmd.Args[0]);
eng.Import(dsList.Length == 0 ? null : dsList);
}
return(0);
WRITE_SYNTAX_ERR:
logError("Invalid commandline: {0}", Environment.CommandLine);
WRITE_SYNTAX:
logError("");
logError("Syntax: <importxml file> [list of datasources] [/flags:<importflags>] [/maxadds:<number>] [/maxemits:<number>] [/$$xxxx$$:<value>");
logError(" or: /resp:<responsefile> with 1 option per line");
return(12);
}
catch (Exception e)
{
logError("Error: {0}\r\nType: {1}\r\nStack:\r\n{2}", e.Message, e.GetType().FullName, e.StackTrace);
return(12);
}
}
public void Load(XmlHelper xml)
{
Xml = xml;
String dir = xml.FileName;
if (!String.IsNullOrEmpty(dir))
{
dir = Path.GetDirectoryName(xml.FileName);
}
Environment.SetEnvironmentVariable("IMPORT_ROOT", dir);
fillTikaVars();
_ImportFlags flags = ImportFlags;
ImportFlags = xml.ReadEnum("@importflags", (_ImportFlags)0);
if ((flags & _ImportFlags.UseFlagsFromXml) == 0)
{
ImportFlags = flags;
}
LogAdds = xml.ReadInt("@logadds", LogAdds);
MaxAdds = xml.ReadInt("@maxadds", MaxAdds);
SwitchesFromXml = xml.ReadStr("@switches", null);
ImportLog.Log("Loading import xml: flags={0}, logadds={1}, maxadds={2}, file={3}.", ImportFlags, LogAdds, MaxAdds, xml.FileName);
binDir = Xml.CombinePath(Xml.ReadStr("@bindir", "bin"));
if (Directory.Exists(binDir))
{
ImportLog.Log(_LogType.ltInfo, "Using extra bin dir: {0}", binDir);
}
else
{
binDir = null;
ImportLog.Log(_LogType.ltInfo, "No bin dir found... All executables are loaded from {0}.\r\nCheck bin dir={1}.", Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), binDir);
}
TempDir = IOUtils.AddSlash(Xml.CombinePath("temp"));
IOUtils.ForceDirectories(TempDir, true);
XmlNode x = Xml.SelectSingleNode("report");
if (x == null)
{
Reporter = null;
}
else
{
Reporter = new MailReporter(x);
}
ImportLog.Log("loaded reporter: {0}", Reporter);
RunAdminSettings = new RunAdministrationSettings(this, Xml.SelectSingleNode("runadmin"));
ImportLog.Log("loaded runadmin settings: {0}", RunAdminSettings);
//Load the supplied script
ImportLog.Log(_LogType.ltTimerStart, "loading: scripts");
XmlNode scriptNode = xml.SelectSingleNode("script");
//Create the holder for the expressions
ScriptExpressions = new ScriptExpressionHolder();
ImportLog.Log(_LogType.ltTimer, "loading: helper process definitions ");
ProcessHostCollection = new ProcessHostCollection(this, xml.SelectSingleNode("processes"));
ImportLog.Log(_LogType.ltTimer, "loading: endpoints");
Endpoints = new Endpoints(this, xml.SelectMandatoryNode("endpoints"));
ImportLog.Log(_LogType.ltTimer, "loading: post-processors");
PostProcessors = new PostProcessors(this, xml.SelectSingleNode("postprocessors"));
ImportLog.Log(_LogType.ltTimer, "loading: converters");
Converters = new Converters(
xml.SelectSingleNode("converters"),
"converter",
(node) => Converter.Create(node),
false);
ImportLog.Log(_LogType.ltTimer, "loading: categories");
Categories = new NamedAdminCollection <CategoryCollection>(
xml.SelectSingleNode("categories"),
"collection",
(node) => new CategoryCollection(node),
true);
ImportLog.Log(_LogType.ltTimer, "loading: pipelines");
Pipelines = new NamedAdminCollection <Pipeline>(
xml.SelectMandatoryNode("pipelines"),
"pipeline",
(node) => new Pipeline(this, node),
true);
ImportLog.Log(_LogType.ltTimer, "loading: datasources");
PipelineContext ctx = new PipelineContext(this);
Datasources = new NamedAdminCollection <DatasourceAdmin>(
xml.SelectMandatoryNode("datasources"),
"datasource",
(node) => new DatasourceAdmin(ctx, node),
true);
//Compile script if needed
if (ScriptExpressions.Count > 0 || scriptNode != null)
{
ScriptHost = new ScriptHost(ScriptHostFlags.Default, TemplateFactory);
if (scriptNode != null)
{
String fn = scriptNode.ReadStr("@file", null);
if (fn != null)
{
ScriptHost.AddFile(xml.CombinePath(fn));
}
}
ScriptHost.ExtraSearchPath = binDir;
if (ScriptExpressions.Count > 0)
{
String fn = TempDir + "_ScriptExpressions.cs";
ScriptExpressions.SaveAndClose(fn);
ScriptHost.AddFile(fn);
}
ScriptHost.AddReference(Assembly.GetExecutingAssembly());
ScriptHost.AddReference(typeof(Bitmanager.Json.JsonExt));
ScriptHost.AddReference(typeof(Bitmanager.Elastic.ESConnection));
ScriptHost.Compile();
}
ImportLog.Log(_LogType.ltTimerStop, "loading: finished. Loaded {0} datasources, {1} pipelines, {2} endpoints, {3} converters, {4} category collections.",
Datasources.Count,
Pipelines.Count,
Endpoints.Count,
Converters.Count,
Categories.Count);
}
