public virtual void Init(PipelineContext ctx, XmlNode node)
{
Init(ctx, node, Encoding.UTF8);
}
protected virtual void _AfterImport(PipelineContext ctx, IDatasourceSink sink)
{
}
protected abstract void ImportStream(PipelineContext ctx, IDatasourceSink sink, IStreamProvider elt, Stream strm);
protected virtual Stream _CreateStream(PipelineContext ctx, IStreamProvider elt)
{
return(elt.CreateStream(ctx));
}
public override Object HandleValue(PipelineContext ctx, String key, Object value)
{
ctx.Pipeline.EmitVariables(ctx, ctx.Pipeline, prefix, splitUntil);
return(value);
}
public override Object HandleValue(PipelineContext ctx, String key, Object value)
{
value = ConvertAndCallScript(ctx, key, value);
Pipeline.SplitInnerTokens(ctx, ctx.Pipeline, (JToken)value, prefix, splitUntil);
return(value);
}
public void Load(XmlHelper xml)
{
Xml = xml;
String dir = xml.FileName;
if (!String.IsNullOrEmpty(dir))
{
dir = Path.GetDirectoryName(xml.FileName);
}
Environment.SetEnvironmentVariable("IMPORT_ROOT", dir);
fillTikaVars();
_ImportFlags flags = ImportFlags;
ImportFlags = xml.ReadEnum("@importflags", (_ImportFlags)0);
if ((flags & _ImportFlags.UseFlagsFromXml) == 0)
{
ImportFlags = flags;
}
LogAdds = xml.ReadInt("@logadds", LogAdds);
MaxAdds = xml.ReadInt("@maxadds", MaxAdds);
SwitchesFromXml = xml.ReadStr("@switches", null);
ImportLog.Log("Loading import xml: flags={0}, logadds={1}, maxadds={2}, file={3}.", ImportFlags, LogAdds, MaxAdds, xml.FileName);
binDir = Xml.CombinePath(Xml.ReadStr("@bindir", "bin"));
if (Directory.Exists(binDir))
{
ImportLog.Log(_LogType.ltInfo, "Using extra bin dir: {0}", binDir);
}
else
{
binDir = null;
ImportLog.Log(_LogType.ltInfo, "No bin dir found... All executables are loaded from {0}.\r\nCheck bin dir={1}.", Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), binDir);
}
TempDir = IOUtils.AddSlash(Xml.CombinePath("temp"));
IOUtils.ForceDirectories(TempDir, true);
XmlNode x = Xml.SelectSingleNode("report");
if (x == null)
{
Reporter = null;
}
else
{
Reporter = new MailReporter(x);
}
ImportLog.Log("loaded reporter: {0}", Reporter);
RunAdminSettings = new RunAdministrationSettings(this, Xml.SelectSingleNode("runadmin"));
ImportLog.Log("loaded runadmin settings: {0}", RunAdminSettings);
//Load the supplied script
ImportLog.Log(_LogType.ltTimerStart, "loading: scripts");
XmlNode scriptNode = xml.SelectSingleNode("script");
//Create the holder for the expressions
ScriptExpressions = new ScriptExpressionHolder();
ImportLog.Log(_LogType.ltTimer, "loading: helper process definitions ");
ProcessHostCollection = new ProcessHostCollection(this, xml.SelectSingleNode("processes"));
ImportLog.Log(_LogType.ltTimer, "loading: endpoints");
Endpoints = new Endpoints(this, xml.SelectMandatoryNode("endpoints"));
ImportLog.Log(_LogType.ltTimer, "loading: post-processors");
PostProcessors = new PostProcessors(this, xml.SelectSingleNode("postprocessors"));
ImportLog.Log(_LogType.ltTimer, "loading: converters");
Converters = new Converters(
xml.SelectSingleNode("converters"),
"converter",
(node) => Converter.Create(node),
false);
ImportLog.Log(_LogType.ltTimer, "loading: categories");
Categories = new NamedAdminCollection <CategoryCollection>(
xml.SelectSingleNode("categories"),
"collection",
(node) => new CategoryCollection(node),
true);
ImportLog.Log(_LogType.ltTimer, "loading: pipelines");
Pipelines = new NamedAdminCollection <Pipeline>(
xml.SelectMandatoryNode("pipelines"),
"pipeline",
(node) => new Pipeline(this, node),
true);
ImportLog.Log(_LogType.ltTimer, "loading: datasources");
PipelineContext ctx = new PipelineContext(this);
Datasources = new NamedAdminCollection <DatasourceAdmin>(
xml.SelectMandatoryNode("datasources"),
"datasource",
(node) => new DatasourceAdmin(ctx, node),
true);
//Compile script if needed
if (ScriptExpressions.Count > 0 || scriptNode != null)
{
ScriptHost = new ScriptHost(ScriptHostFlags.Default, TemplateFactory);
if (scriptNode != null)
{
String fn = scriptNode.ReadStr("@file", null);
if (fn != null)
{
ScriptHost.AddFile(xml.CombinePath(fn));
}
}
ScriptHost.ExtraSearchPath = binDir;
if (ScriptExpressions.Count > 0)
{
String fn = TempDir + "_ScriptExpressions.cs";
ScriptExpressions.SaveAndClose(fn);
ScriptHost.AddFile(fn);
}
ScriptHost.AddReference(Assembly.GetExecutingAssembly());
ScriptHost.AddReference(typeof(Bitmanager.Json.JsonExt));
ScriptHost.AddReference(typeof(Bitmanager.Elastic.ESConnection));
ScriptHost.Compile();
}
ImportLog.Log(_LogType.ltTimerStop, "loading: finished. Loaded {0} datasources, {1} pipelines, {2} endpoints, {3} converters, {4} category collections.",
Datasources.Count,
Pipelines.Count,
Endpoints.Count,
Converters.Count,
Categories.Count);
}
public ImportReport Import(String[] enabledDSses = null)
{
var ret = new ImportReport();
RunAdministrations = RunAdminSettings.Load();
StartTimeUtc = DateTime.UtcNow;
ImportLog.Log();
ImportLog.Log(new String('_', 80));
ImportLog.Log(_LogType.ltProgress, "Starting import. VirtMem={3:F1}GB, Flags={0}, MaxAdds={1}, ActiveDS's='{2}'.", ImportFlags, MaxAdds, enabledDSses == null ? null : String.Join(", ", enabledDSses), OS.GetTotalVirtualMemory() / (1024 * 1024 * 1024.0));
PipelineContext mainCtx = new PipelineContext(this);
try
{
_ErrorState stateFilter = _ErrorState.All;
if ((ImportFlags & _ImportFlags.IgnoreLimited) != 0)
{
stateFilter &= ~_ErrorState.Limited;
}
if ((ImportFlags & _ImportFlags.IgnoreErrors) != 0)
{
stateFilter &= ~_ErrorState.Error;
}
Endpoints.Open(mainCtx);
for (int i = 0; i < Datasources.Count; i++)
{
DatasourceAdmin admin = Datasources[i];
if (!String.IsNullOrEmpty(OverrideEndpoint))
{
ImportLog.Log(_LogType.ltWarning, "Datsource {0} will run with the override endpoint={1}", admin.Name, OverrideEndpoint);
admin.EndpointName = OverrideEndpoint;
}
if (!String.IsNullOrEmpty(OverrideEndpoint))
{
ImportLog.Log(_LogType.ltWarning, "Datsource {0} will run with the override pipeline={1}", admin.Name, OverridePipeline);
//admin.p = OverrideEndpoint;
}
if (!isActive(enabledDSses, admin))
{
ImportLog.Log(_LogType.ltProgress, "[{0}]: not active", admin.Name);
continue;
}
var report = new DatasourceReport(admin);
ret.Add(report);
PipelineContext ctx = new PipelineContext(mainCtx, admin, report);
var pipeline = admin.Pipeline;
try
{
admin.Import(ctx);
mainCtx.ErrorState |= (ctx.ErrorState & stateFilter);
if (ctx.LastError != null)
{
mainCtx.LastError = ctx.LastError;
}
report.MarkEnded(ctx);
}
catch (Exception err)
{
mainCtx.LastError = err;
mainCtx.ErrorState |= (ctx.ErrorState & stateFilter) | _ErrorState.Error;
report.MarkEnded(ctx);
throw;
}
Endpoints.OptClosePerDatasource(ctx);
foreach (var c in Converters)
{
c.DumpMissed(ctx);
}
}
ImportLog.Log(_LogType.ltProgress, "Import ended");
ProcessHostCollection.StopAll();
Endpoints.Close(mainCtx);
RunAdminSettings.Save(RunAdministrations);
}
catch (Exception err2)
{
mainCtx.LastError = err2;
throw;
}
finally
{
try
{
Endpoints.CloseFinally(mainCtx);
}
catch (Exception e2)
{
ErrorLog.Log(e2);
ImportLog.Log(e2);
}
try
{
ProcessHostCollection.StopAll();
}
catch (Exception e2)
{
ErrorLog.Log(e2);
ImportLog.Log(e2);
}
try
{
ret.SetGlobalStatus(mainCtx);
if (Reporter != null)
{
Reporter.SendReport(mainCtx, ret);
}
}
catch (Exception e2)
{
ErrorLog.Log(e2);
ImportLog.Log(e2);
}
}
//ret.SetGlobalStatus(mainCtx);
ImportLog.Log("Unknown switches: [{0}]", ret.UnknownSwitches);
ImportLog.Log("Mentioned switches: [{0}]", ret.MentionedSwitches);
return(ret);
}
public ContextCallback(PipelineContext ctx, ESDatasource ds, IStreamProvider elt)
{
this.elt = elt;
this.ctx = ctx;
this.ds = ds;
}
private void importUrl(PipelineContext ctx, IDatasourceSink sink, IStreamProvider elt)
{
int maxParallel = elt.ContextNode.ReadInt("@maxparallel", this.maxParallel);
int splitUntil = elt.ContextNode.ReadInt("@splituntil", this.splitUntil);
if (splitUntil < 0)
{
splitUntil = int.MaxValue;
}
bool scan = elt.ContextNode.ReadBool("@scan", this.scan);
String url = elt.ToString();
ctx.SendItemStart(elt);
String command = elt.ContextNode.ReadStr("@command", null);
String index = command != null ? null : elt.ContextNode.ReadStr("@index"); //mutual exclusive with command
String reqBody = elt.ContextNode.ReadStr("request", this.requestBody);
JObject req = null;
if (reqBody != null)
{
req = JObject.Parse(reqBody);
}
ctx.DebugLog.Log("Request scan={1}, body={0}", reqBody, scan);
try
{
Uri uri = new Uri(url);
ESConnection conn = ESHelper.CreateConnection(ctx, url);
ContextCallback cb = new ContextCallback(ctx, this, elt);
conn.Timeout = timeoutInMs; //Same timeout as what we send to ES
conn.OnPrepareRequest = cb.OnPrepareRequest;
if (command != null)
{
var resp = conn.SendCmd("POST", command, reqBody);
resp.ThrowIfError();
Pipeline.EmitToken(ctx, sink, resp.JObject, "response", splitUntil);
}
else
{
ESRecordEnum e = new ESRecordEnum(conn, index, req, numRecords, timeout, scan);
if (maxParallel > 0)
{
e.Async = true;
}
ctx.ImportLog.Log("Starting scan of {0} records. Index={1}, connection={2}, async={3}, buffersize={4} requestbody={5}, splituntil={6}, scan={7}.", e.Count, index, url, e.Async, numRecords, req != null, splitUntil, scan);
foreach (var doc in e)
{
ctx.IncrementEmitted();
sink.HandleValue(ctx, "record/_sort", doc.Sort);
sink.HandleValue(ctx, "record/_type", doc.Type);
if (splitUntil != 0)
{
foreach (var kvp in doc)
{
String pfx = "record/" + kvp.Key;
if (splitUntil == 1)
{
sink.HandleValue(ctx, pfx, kvp.Value);
continue;
}
Pipeline.EmitToken(ctx, sink, kvp.Value, pfx, splitUntil - 1);
}
}
sink.HandleValue(ctx, "record", doc);
}
}
ctx.SendItemStop();
}
catch (Exception e)
{
ctx.HandleException(e);
}
}
public abstract void ProcessRecord(PipelineContext ctx, JObject rec);
public virtual IRecordAction Clone(PipelineContext ctx)
{
return(this);
}
public override void ProcessRecord(PipelineContext ctx, JObject record)
{
scriptDelegate(ctx, record);
}
public override IRecordAction Clone(PipelineContext ctx)
{
return(new RecordScriptAction(ctx, this));
}
protected RecordScriptAction(PipelineContext ctx, RecordScriptAction other)
{
ScriptName = other.ScriptName;
scriptDelegate = ctx.Pipeline.CreateScriptDelegate <ScriptDelegate>(ScriptName);
}